skillar 0.0.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -0
- package/README.md +193 -1
- package/dist/analysis/finding.js +190 -0
- package/dist/analysis/tree.js +157 -0
- package/dist/analysis/usage.js +394 -0
- package/dist/api/flows.js +285 -0
- package/dist/api/routes.js +192 -0
- package/dist/api/security.js +86 -0
- package/dist/api/sse.js +55 -0
- package/dist/api/static.js +33 -0
- package/dist/catalog/fields.js +159 -0
- package/dist/catalog/index.js +133 -0
- package/dist/catalog/scanner.js +204 -0
- package/dist/catalog/uri.js +43 -0
- package/dist/catalog/watcher.js +59 -0
- package/dist/cli/config-writer.js +255 -0
- package/dist/cli/doctor.js +268 -0
- package/dist/cli/eval.js +64 -0
- package/dist/cli/export-import.js +68 -0
- package/dist/cli/init.js +272 -0
- package/dist/cli/main.js +165 -0
- package/dist/cli/open.js +24 -0
- package/dist/cli/serve.js +147 -0
- package/dist/config.js +92 -0
- package/dist/dashboard/assets/app.js +354 -0
- package/dist/dashboard/assets/chunk-5DFWNJC2.js +1 -0
- package/dist/dashboard/assets/chunk-S6B2HJQA.js +17 -0
- package/dist/dashboard/assets/styles.css +542 -0
- package/dist/dashboard/index.html +15 -0
- package/dist/fs-watchdog.js +35 -0
- package/dist/herdr/client.js +205 -0
- package/dist/herdr/sync.js +342 -0
- package/dist/ingest/attribution.js +80 -0
- package/dist/ingest/hooks.js +168 -0
- package/dist/ingest/identity.js +4 -0
- package/dist/ingest/mcp-recorder.js +41 -0
- package/dist/ingest/otlp-traces.js +106 -0
- package/dist/ingest/otlp.js +314 -0
- package/dist/ingest/pipeline.js +119 -0
- package/dist/ingest/redact.js +36 -0
- package/dist/ingest/runtimes.js +25 -0
- package/dist/mcp/dispatch.js +54 -0
- package/dist/mcp/http.js +112 -0
- package/dist/mcp/legacy.js +61 -0
- package/dist/mcp/pagination.js +17 -0
- package/dist/mcp/protocol.js +121 -0
- package/dist/mcp/skills-extension.js +77 -0
- package/dist/mcp/stdio-proxy.js +108 -0
- package/dist/mcp/tools.js +59 -0
- package/dist/packs/claude-code.js +176 -0
- package/dist/packs/codex.js +207 -0
- package/dist/packs/cursor.js +97 -0
- package/dist/packs/gajae.js +7 -0
- package/dist/packs/gemini.js +90 -0
- package/dist/packs/json-config.js +55 -0
- package/dist/packs/omo.js +154 -0
- package/dist/packs/openclaw.js +132 -0
- package/dist/packs/registry.js +13 -0
- package/dist/search/fts.js +61 -0
- package/dist/search/fusion.js +50 -0
- package/dist/search/guidance.js +7 -0
- package/dist/search/index.js +74 -0
- package/dist/search/lexical.js +69 -0
- package/dist/server.js +267 -0
- package/dist/store/automation.js +18 -0
- package/dist/store/db.js +134 -0
- package/dist/store/execution.js +157 -0
- package/dist/store/export.js +269 -0
- package/dist/store/index.js +14 -0
- package/dist/store/migrations/002_tracing.sql +94 -0
- package/dist/store/migrations/003_tracing_indexes.sql +11 -0
- package/dist/store/migrations/004_load_reconciliation.sql +50 -0
- package/dist/store/migrations/005_execution_boundaries.sql +6 -0
- package/dist/store/migrations/006_recording_order.sql +72 -0
- package/dist/store/queries.js +643 -0
- package/dist/store/schema.sql +96 -0
- package/dist/store/workflow.js +250 -0
- package/dist/store/writes.js +315 -0
- package/dist/types.js +1 -0
- package/dist/version.js +19 -0
- package/dist/workflow/compare.js +68 -0
- package/dist/workflow/routes.js +100 -0
- package/dist/workflow/spec.js +40 -0
- package/docs/execution-ownership.md +98 -0
- package/docs/mcp-conformance.md +156 -0
- package/docs/skill-tracing.md +294 -0
- package/package.json +46 -3
- package/packs/claude-code/README.md +5 -0
- package/packs/claude-code/settings.template.json +34 -0
- package/packs/codex/README.md +8 -0
- package/packs/codex/config.template.toml +12 -0
- package/packs/cursor/hooks.example.json +8 -0
- package/packs/cursor/mcp.example.json +8 -0
- package/packs/gajae/extension/discovery.md +5 -0
- package/packs/gajae/extension/index.js +52 -0
- package/packs/gemini/settings.example.json +16 -0
- package/packs/hook-post.mjs +37 -0
- package/packs/omo/extension/discovery.md +5 -0
- package/packs/omo/extension/index.js +52 -0
- package/packs/openclaw/.codex-plugin/plugin.json +11 -0
- package/packs/openclaw/.mcp.json +12 -0
- package/packs/openclaw/hooks/skillar-discovery/HOOK.md +8 -0
- package/packs/openclaw/hooks/skillar-discovery/handler.js +30 -0
- package/packs/openclaw/index.mjs +55 -0
- package/packs/openclaw/openclaw.plugin.json +22 -0
- package/packs/openclaw/package.json +11 -0
- package/packs/openclaw/skillar.json +1 -0
- package/src/store/migrations/002_tracing.sql +94 -0
- package/src/store/migrations/003_tracing_indexes.sql +11 -0
- package/src/store/migrations/004_load_reconciliation.sql +50 -0
- package/src/store/migrations/005_execution_boundaries.sql +6 -0
- package/src/store/migrations/006_recording_order.sql +72 -0
- package/src/store/schema.sql +96 -0
- package/src/store/task-types.json +15 -0
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { homedir } from "node:os";
|
|
3
|
+
import { join, resolve } from "node:path";
|
|
4
|
+
import { DEFAULT_ROOTS, expandHome } from "../config.js";
|
|
5
|
+
import { parseJson, readAliasesOverrides, storeError, writeAliasesOverrides } from "./writes.js";
|
|
6
|
+
export const EXPORT_FILE = "skillar-export.json";
|
|
7
|
+
export const HISTORY_FILE = "history.db";
|
|
8
|
+
/**
|
|
9
|
+
* Tables that export --with-history carries, in foreign-key order, each with the columns identifying a row.
|
|
10
|
+
* A table or column missing on either side (importing a v1 history.db into a v2 store) is simply not copied.
|
|
11
|
+
*/
|
|
12
|
+
const HISTORY_TABLES = {
|
|
13
|
+
sessions: ["id"],
|
|
14
|
+
events: ["id"],
|
|
15
|
+
loads: ["load_id"],
|
|
16
|
+
otel_spans: ["trace_id", "span_id"],
|
|
17
|
+
herdr_workspaces: ["id"],
|
|
18
|
+
herdr_tabs: ["id"],
|
|
19
|
+
herdr_panes: ["id"],
|
|
20
|
+
pane_sessions: ["pane_id", "agent_session_value"],
|
|
21
|
+
workflow_specs: ["id"],
|
|
22
|
+
flows: ["id"],
|
|
23
|
+
flow_steps: ["flow_id", "pane_id"],
|
|
24
|
+
};
|
|
25
|
+
// recording_clock is deliberately local. Row provenance and flow cuts travel
|
|
26
|
+
// through shared columns; insert triggers add destination arrival provenance.
|
|
27
|
+
const HISTORY_VERSIONS = [1, 2, 3, 4, 5, 6];
|
|
28
|
+
function tildify(path, home) {
|
|
29
|
+
const h = home.replace(/\/+$/, "");
|
|
30
|
+
if (path === h)
|
|
31
|
+
return "~";
|
|
32
|
+
if (path.startsWith(h + "/"))
|
|
33
|
+
return "~/" + path.slice(h.length + 1);
|
|
34
|
+
return path;
|
|
35
|
+
}
|
|
36
|
+
function readConfigFile(skillarHome) {
|
|
37
|
+
if (!skillarHome)
|
|
38
|
+
return undefined;
|
|
39
|
+
const p = join(skillarHome, "config.json");
|
|
40
|
+
return existsSync(p) ? parseJson(readFileSync(p, "utf8"), {}) : undefined;
|
|
41
|
+
}
|
|
42
|
+
export function createPortable(ctx) {
|
|
43
|
+
const { db } = ctx;
|
|
44
|
+
const home = () => ctx.opts.home ?? homedir();
|
|
45
|
+
function existingEvalKeys() {
|
|
46
|
+
const rows = db.prepare("SELECT query, expected_uri FROM eval_cases").all();
|
|
47
|
+
return new Set(rows.map((r) => JSON.stringify([r.query, r.expected_uri])));
|
|
48
|
+
}
|
|
49
|
+
function columnsOf(schema, table) {
|
|
50
|
+
return db.prepare(`PRAGMA ${schema}.table_info(${table})`).all().map((c) => c.name);
|
|
51
|
+
}
|
|
52
|
+
function historyShape() {
|
|
53
|
+
const shape = [];
|
|
54
|
+
for (const [table, key] of Object.entries(HISTORY_TABLES)) {
|
|
55
|
+
const mine = new Set(columnsOf("main", table));
|
|
56
|
+
const columns = columnsOf("hist", table).filter((c) => mine.has(c));
|
|
57
|
+
if (key.every((k) => columns.includes(k)))
|
|
58
|
+
shape.push({ table, key, columns });
|
|
59
|
+
}
|
|
60
|
+
return shape;
|
|
61
|
+
}
|
|
62
|
+
const keyIn = (table, key) => `${key.length === 1 ? key[0] : `(${key.join(", ")})`} IN (SELECT ${key.join(", ")} FROM main.${table})`;
|
|
63
|
+
/**
|
|
64
|
+
* A history row the destination already holds: the same primary key, the same agent record stored under another event
|
|
65
|
+
* id (source_key), or the same workflow spec version registered there independently.
|
|
66
|
+
*/
|
|
67
|
+
function present(table, key, columns) {
|
|
68
|
+
if (table === "events" && columns.includes("source_key")) {
|
|
69
|
+
return `(${keyIn(table, key)} OR (source_key IS NOT NULL AND source_key IN (SELECT source_key FROM main.events WHERE source_key IS NOT NULL)))`;
|
|
70
|
+
}
|
|
71
|
+
if (table === "workflow_specs")
|
|
72
|
+
return `(${keyIn(table, key)} OR (name, version) IN (SELECT name, version FROM main.workflow_specs))`;
|
|
73
|
+
return keyIn(table, key);
|
|
74
|
+
}
|
|
75
|
+
/** Two different bodies under one spec name and version cannot both be kept, and picking one would rewrite flows. */
|
|
76
|
+
function refuseSpecConflicts(shape) {
|
|
77
|
+
if (!shape.some((s) => s.table === "workflow_specs"))
|
|
78
|
+
return;
|
|
79
|
+
const clash = db.prepare(`SELECT h.name, h.version FROM hist.workflow_specs h JOIN main.workflow_specs m
|
|
80
|
+
ON m.name = h.name AND m.version = h.version WHERE m.digest <> h.digest ORDER BY h.name, h.version LIMIT 1`).get();
|
|
81
|
+
if (clash) {
|
|
82
|
+
throw storeError("history_spec_conflict", `workflow spec ${clash.name} v${clash.version} differs between this store and the history`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
function historyPlan() {
|
|
86
|
+
const tables = {};
|
|
87
|
+
let skipped = 0;
|
|
88
|
+
const shape = historyShape();
|
|
89
|
+
refuseSpecConflicts(shape);
|
|
90
|
+
for (const { table, key, columns } of shape) {
|
|
91
|
+
const row = db.prepare(`SELECT SUM(CASE WHEN ${present(table, key, columns)} THEN 0 ELSE 1 END) AS fresh, COUNT(*) AS total FROM hist.${table}`).get();
|
|
92
|
+
tables[table] = row.fresh ?? 0;
|
|
93
|
+
skipped += row.total - (row.fresh ?? 0);
|
|
94
|
+
}
|
|
95
|
+
return { tables, skipped };
|
|
96
|
+
}
|
|
97
|
+
function copyHistory() {
|
|
98
|
+
for (const { table, key, columns } of historyShape()) {
|
|
99
|
+
const cols = columns.join(", ");
|
|
100
|
+
// A flow follows its spec to the id the destination keeps for that name and version.
|
|
101
|
+
const values = columns.map((c) => table === "flows" && c === "spec_id"
|
|
102
|
+
? `COALESCE((SELECT m.id FROM main.workflow_specs m JOIN hist.workflow_specs h ON h.name = m.name AND h.version = m.version
|
|
103
|
+
WHERE h.id = src.spec_id), src.spec_id)`
|
|
104
|
+
: `src.${c}`).join(", ");
|
|
105
|
+
db.prepare(`INSERT INTO main.${table} (${cols}) SELECT ${values} FROM hist.${table} AS src WHERE NOT ${present(table, key, columns)}`).run();
|
|
106
|
+
// Legacy loads have no snapshot: insert triggers may run before later-received activity rows.
|
|
107
|
+
// Complete their candidate index once the whole events table is available, not on report reads.
|
|
108
|
+
if (table === "events")
|
|
109
|
+
db.exec(`INSERT OR IGNORE INTO load_candidates
|
|
110
|
+
SELECT * FROM load_candidate_rows WHERE load_id IN (SELECT id FROM hist.events)`);
|
|
111
|
+
}
|
|
112
|
+
// Restoring newer history into an older backup of the SAME lineage must also
|
|
113
|
+
// advance past deleted observations. Never adopt another database's domain.
|
|
114
|
+
if (columnsOf("hist", "recording_clock").includes("position"))
|
|
115
|
+
db.exec(`UPDATE recording_clock
|
|
116
|
+
SET position = MAX(position, COALESCE((SELECT h.position FROM hist.recording_clock h
|
|
117
|
+
WHERE h.domain = recording_clock.domain AND h.singleton = 1), 0)) WHERE singleton = 1`);
|
|
118
|
+
}
|
|
119
|
+
return {
|
|
120
|
+
exportTo(dir, opts) {
|
|
121
|
+
mkdirSync(dir, { recursive: true });
|
|
122
|
+
const raw = readConfigFile(ctx.opts.skillarHome);
|
|
123
|
+
const source = raw ?? (ctx.opts.config ? { ...ctx.opts.config } : {});
|
|
124
|
+
const { token: _token, roots: rawRoots, ...config } = source;
|
|
125
|
+
void _token;
|
|
126
|
+
const roots = (Array.isArray(rawRoots) ? rawRoots : [])
|
|
127
|
+
.map((r) => ({ id: r.id, path: tildify(r.path, home()) }));
|
|
128
|
+
const evalRows = db.prepare("SELECT query, expected_uri, from_trace_id FROM eval_cases ORDER BY created_at, rowid").all();
|
|
129
|
+
const doc = {
|
|
130
|
+
version: 1,
|
|
131
|
+
config,
|
|
132
|
+
roots,
|
|
133
|
+
evalCases: evalRows.map((r) => ({ query: r.query, ...(r.expected_uri !== null ? { expectedUri: r.expected_uri } : {}),
|
|
134
|
+
...(r.from_trace_id !== null ? { fromTraceId: r.from_trace_id } : {}) })),
|
|
135
|
+
aliasesOverrides: readAliasesOverrides(db),
|
|
136
|
+
};
|
|
137
|
+
const out = join(dir, EXPORT_FILE);
|
|
138
|
+
writeFileSync(out, JSON.stringify(doc, null, 2) + "\n", { mode: 0o600 });
|
|
139
|
+
chmodSync(out, 0o600);
|
|
140
|
+
const history = join(dir, HISTORY_FILE);
|
|
141
|
+
rmSync(history, { force: true });
|
|
142
|
+
if (opts.withHistory) {
|
|
143
|
+
db.prepare("VACUUM INTO ?").run(resolve(history));
|
|
144
|
+
chmodSync(history, 0o600);
|
|
145
|
+
}
|
|
146
|
+
},
|
|
147
|
+
importFrom(dir, opts) {
|
|
148
|
+
const file = existsSync(join(dir, EXPORT_FILE)) ? join(dir, EXPORT_FILE) : join(dir, "gisool-export.json");
|
|
149
|
+
if (!existsSync(file))
|
|
150
|
+
throw storeError("no_export_found", `no ${EXPORT_FILE} in ${dir}`);
|
|
151
|
+
const doc = parseJson(readFileSync(file, "utf8"), {});
|
|
152
|
+
if (doc.version !== 1)
|
|
153
|
+
throw storeError("unsupported_export_version", `unsupported export version: ${String(doc.version)}`);
|
|
154
|
+
const apply = opts.merge;
|
|
155
|
+
const evalKeys = existingEvalKeys();
|
|
156
|
+
const evalToAdd = [];
|
|
157
|
+
let evalSkipped = 0;
|
|
158
|
+
for (const c of doc.evalCases ?? []) {
|
|
159
|
+
const key = JSON.stringify([c.query, c.expectedUri ?? null]);
|
|
160
|
+
if (evalKeys.has(key))
|
|
161
|
+
evalSkipped++;
|
|
162
|
+
else {
|
|
163
|
+
evalKeys.add(key);
|
|
164
|
+
evalToAdd.push(c);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
const currentAliases = readAliasesOverrides(db);
|
|
168
|
+
const aliasAdded = Object.keys(doc.aliasesOverrides ?? {}).filter((k) => !(k in currentAliases));
|
|
169
|
+
const aliasSkipped = Object.keys(doc.aliasesOverrides ?? {}).filter((k) => k in currentAliases);
|
|
170
|
+
const skillarHome = ctx.opts.skillarHome;
|
|
171
|
+
const existingRaw = readConfigFile(skillarHome);
|
|
172
|
+
const configBase = existingRaw ?? {};
|
|
173
|
+
const existingRoots = (Array.isArray(configBase.roots) ? configBase.roots : existingRaw ? [] : DEFAULT_ROOTS);
|
|
174
|
+
const rootsAdded = [];
|
|
175
|
+
const rootsSkipped = [];
|
|
176
|
+
const configAdded = [];
|
|
177
|
+
const configKept = [];
|
|
178
|
+
if (skillarHome) {
|
|
179
|
+
const knownPaths = new Set(existingRoots.map((r) => expandHome(r.path, home())));
|
|
180
|
+
const knownIds = new Set(existingRoots.map((r) => r.id));
|
|
181
|
+
for (const r of doc.roots ?? []) {
|
|
182
|
+
const expanded = expandHome(r.path, home());
|
|
183
|
+
if (knownIds.has(r.id) || knownPaths.has(expanded))
|
|
184
|
+
rootsSkipped.push(r.id);
|
|
185
|
+
else {
|
|
186
|
+
rootsAdded.push({ id: r.id, path: tildify(expanded, home()) });
|
|
187
|
+
knownIds.add(r.id);
|
|
188
|
+
knownPaths.add(expanded);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
for (const k of Object.keys(doc.config ?? {})) {
|
|
192
|
+
if (k === "token" || k === "roots")
|
|
193
|
+
continue;
|
|
194
|
+
if (k in configBase)
|
|
195
|
+
configKept.push(k);
|
|
196
|
+
else
|
|
197
|
+
configAdded.push(k);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
const historyPath = join(dir, HISTORY_FILE);
|
|
201
|
+
let history = null;
|
|
202
|
+
const hasHistory = existsSync(historyPath);
|
|
203
|
+
if (hasHistory) {
|
|
204
|
+
db.prepare("ATTACH DATABASE ? AS hist").run(resolve(historyPath));
|
|
205
|
+
try {
|
|
206
|
+
const version = db.prepare("PRAGMA hist.user_version").get().user_version;
|
|
207
|
+
if (!HISTORY_VERSIONS.includes(version))
|
|
208
|
+
throw storeError("unsupported_history_version", `history.db user_version ${version}`);
|
|
209
|
+
history = historyPlan();
|
|
210
|
+
if (apply) {
|
|
211
|
+
db.exec("BEGIN IMMEDIATE");
|
|
212
|
+
try {
|
|
213
|
+
copyHistory();
|
|
214
|
+
db.exec("COMMIT");
|
|
215
|
+
}
|
|
216
|
+
catch (error) {
|
|
217
|
+
db.exec("ROLLBACK");
|
|
218
|
+
throw error;
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
finally {
|
|
223
|
+
db.exec("DETACH DATABASE hist");
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
if (apply) {
|
|
227
|
+
const insert = db.prepare("INSERT INTO eval_cases (id, query, expected_uri, from_trace_id, created_at) VALUES (lower(hex(randomblob(16))), ?, ?, ?, ?)");
|
|
228
|
+
db.exec("BEGIN IMMEDIATE");
|
|
229
|
+
try {
|
|
230
|
+
for (const c of evalToAdd)
|
|
231
|
+
insert.run(c.query, c.expectedUri ?? null, c.fromTraceId ?? null, ctx.now());
|
|
232
|
+
if (aliasAdded.length > 0) {
|
|
233
|
+
const next = { ...currentAliases };
|
|
234
|
+
for (const k of aliasAdded)
|
|
235
|
+
next[k] = doc.aliasesOverrides[k];
|
|
236
|
+
writeAliasesOverrides(db, next);
|
|
237
|
+
}
|
|
238
|
+
db.exec("COMMIT");
|
|
239
|
+
}
|
|
240
|
+
catch (error) {
|
|
241
|
+
db.exec("ROLLBACK");
|
|
242
|
+
throw error;
|
|
243
|
+
}
|
|
244
|
+
if (skillarHome && (rootsAdded.length > 0 || configAdded.length > 0)) {
|
|
245
|
+
const next = { ...configBase };
|
|
246
|
+
for (const k of configAdded)
|
|
247
|
+
next[k] = doc.config[k];
|
|
248
|
+
next.roots = [...existingRoots, ...rootsAdded];
|
|
249
|
+
mkdirSync(skillarHome, { recursive: true, mode: 0o700 });
|
|
250
|
+
const p = join(skillarHome, "config.json");
|
|
251
|
+
writeFileSync(p, JSON.stringify(next, null, 2) + "\n", { mode: 0o600 });
|
|
252
|
+
chmodSync(p, 0o600);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
const historyAdded = history ? Object.values(history.tables).reduce((sum, n) => sum + n, 0) : 0;
|
|
256
|
+
return {
|
|
257
|
+
added: evalToAdd.length + rootsAdded.length + configAdded.length + aliasAdded.length + historyAdded,
|
|
258
|
+
skipped: evalSkipped + rootsSkipped.length + configKept.length + aliasSkipped.length + (history?.skipped ?? 0),
|
|
259
|
+
details: {
|
|
260
|
+
evalCases: { added: evalToAdd.length, skipped: evalSkipped },
|
|
261
|
+
roots: { added: rootsAdded.map((r) => r.id), skipped: rootsSkipped },
|
|
262
|
+
configKeys: { added: configAdded, kept: configKept },
|
|
263
|
+
aliases: { added: aliasAdded, skipped: aliasSkipped },
|
|
264
|
+
history,
|
|
265
|
+
},
|
|
266
|
+
};
|
|
267
|
+
},
|
|
268
|
+
};
|
|
269
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { createPortable } from "./export.js";
|
|
2
|
+
import { createQueries } from "./queries.js";
|
|
3
|
+
import { createWorkflow } from "./workflow.js";
|
|
4
|
+
import { createWrites } from "./writes.js";
|
|
5
|
+
export { EXPORT_FILE, HISTORY_FILE } from "./export.js";
|
|
6
|
+
export { adoptionState, classifyTaskType, localDowHour, taskTypeRules } from "./queries.js";
|
|
7
|
+
/**
|
|
8
|
+
* importFrom(dir, { merge: false }) is a dry run: it returns the same counts (plus `details`) without writing anything;
|
|
9
|
+
* { merge: true } applies, never overwriting existing config keys, roots, aliases, eval cases, sessions, or events.
|
|
10
|
+
*/
|
|
11
|
+
export function createStore(db, opts = {}) {
|
|
12
|
+
const ctx = { db, opts, now: opts.now ?? Date.now };
|
|
13
|
+
return { ...createWrites(ctx), ...createQueries(ctx), ...createPortable(ctx), ...createWorkflow(ctx), close: () => db.close() };
|
|
14
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
-- skillar store migration 2: tracing keys on events, herdr topology and workflow specs (user_version 1 -> 2).
|
|
2
|
+
-- db.ts runs this inside a transaction and sets user_version afterwards, so this file sets no PRAGMA.
|
|
3
|
+
|
|
4
|
+
ALTER TABLE events ADD COLUMN source_key TEXT;
|
|
5
|
+
ALTER TABLE events ADD COLUMN turn_key TEXT;
|
|
6
|
+
ALTER TABLE events ADD COLUMN call_id TEXT;
|
|
7
|
+
ALTER TABLE events ADD COLUMN span_id TEXT;
|
|
8
|
+
|
|
9
|
+
-- source_key identifies the agent record an event was derived from, so replaying a batch stores nothing new.
|
|
10
|
+
CREATE UNIQUE INDEX IF NOT EXISTS events_source_key ON events(source_key) WHERE source_key IS NOT NULL;
|
|
11
|
+
CREATE INDEX IF NOT EXISTS events_turn ON events(session_id, turn_key);
|
|
12
|
+
CREATE INDEX IF NOT EXISTS events_call ON events(call_id) WHERE call_id IS NOT NULL;
|
|
13
|
+
|
|
14
|
+
CREATE TABLE IF NOT EXISTS otel_spans (
|
|
15
|
+
trace_id TEXT NOT NULL,
|
|
16
|
+
span_id TEXT NOT NULL,
|
|
17
|
+
parent_span_id TEXT,
|
|
18
|
+
name TEXT NOT NULL,
|
|
19
|
+
runtime TEXT NOT NULL,
|
|
20
|
+
conversation_id TEXT,
|
|
21
|
+
turn_key TEXT,
|
|
22
|
+
call_id TEXT,
|
|
23
|
+
start_ms INTEGER,
|
|
24
|
+
end_ms INTEGER,
|
|
25
|
+
received_at INTEGER NOT NULL,
|
|
26
|
+
PRIMARY KEY (trace_id, span_id)
|
|
27
|
+
);
|
|
28
|
+
CREATE INDEX IF NOT EXISTS otel_spans_call ON otel_spans(call_id);
|
|
29
|
+
CREATE INDEX IF NOT EXISTS otel_spans_turn ON otel_spans(turn_key);
|
|
30
|
+
|
|
31
|
+
CREATE TABLE IF NOT EXISTS herdr_workspaces (
|
|
32
|
+
id TEXT PRIMARY KEY,
|
|
33
|
+
label TEXT,
|
|
34
|
+
created_ms INTEGER,
|
|
35
|
+
closed_ms INTEGER
|
|
36
|
+
);
|
|
37
|
+
|
|
38
|
+
CREATE TABLE IF NOT EXISTS herdr_tabs (
|
|
39
|
+
id TEXT PRIMARY KEY,
|
|
40
|
+
workspace_id TEXT,
|
|
41
|
+
label TEXT,
|
|
42
|
+
created_ms INTEGER,
|
|
43
|
+
closed_ms INTEGER
|
|
44
|
+
);
|
|
45
|
+
|
|
46
|
+
CREATE TABLE IF NOT EXISTS herdr_panes (
|
|
47
|
+
id TEXT PRIMARY KEY,
|
|
48
|
+
workspace_id TEXT,
|
|
49
|
+
tab_id TEXT,
|
|
50
|
+
label TEXT,
|
|
51
|
+
cwd TEXT,
|
|
52
|
+
created_ms INTEGER,
|
|
53
|
+
closed_ms INTEGER
|
|
54
|
+
);
|
|
55
|
+
|
|
56
|
+
-- session_id stays NULL until a skillar session matches the reported agent session value.
|
|
57
|
+
CREATE TABLE IF NOT EXISTS pane_sessions (
|
|
58
|
+
pane_id TEXT NOT NULL,
|
|
59
|
+
agent TEXT,
|
|
60
|
+
source TEXT,
|
|
61
|
+
agent_session_value TEXT NOT NULL,
|
|
62
|
+
session_id TEXT REFERENCES sessions(id),
|
|
63
|
+
first_seen_ms INTEGER,
|
|
64
|
+
last_seen_ms INTEGER,
|
|
65
|
+
PRIMARY KEY (pane_id, agent_session_value)
|
|
66
|
+
);
|
|
67
|
+
|
|
68
|
+
CREATE TABLE IF NOT EXISTS workflow_specs (
|
|
69
|
+
id TEXT PRIMARY KEY,
|
|
70
|
+
name TEXT NOT NULL,
|
|
71
|
+
version INTEGER NOT NULL,
|
|
72
|
+
body_json TEXT NOT NULL,
|
|
73
|
+
digest TEXT NOT NULL,
|
|
74
|
+
created_ms INTEGER NOT NULL,
|
|
75
|
+
UNIQUE (name, version)
|
|
76
|
+
);
|
|
77
|
+
|
|
78
|
+
CREATE TABLE IF NOT EXISTS flows (
|
|
79
|
+
id TEXT PRIMARY KEY,
|
|
80
|
+
workspace_id TEXT,
|
|
81
|
+
tab_id TEXT,
|
|
82
|
+
spec_id TEXT REFERENCES workflow_specs(id),
|
|
83
|
+
label TEXT,
|
|
84
|
+
registered_ms INTEGER,
|
|
85
|
+
ended_ms INTEGER
|
|
86
|
+
);
|
|
87
|
+
|
|
88
|
+
CREATE TABLE IF NOT EXISTS flow_steps (
|
|
89
|
+
flow_id TEXT NOT NULL REFERENCES flows(id),
|
|
90
|
+
pane_id TEXT NOT NULL,
|
|
91
|
+
step_id TEXT NOT NULL,
|
|
92
|
+
registered_ms INTEGER,
|
|
93
|
+
PRIMARY KEY (flow_id, pane_id)
|
|
94
|
+
);
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
-- Bounded pane lifetime searches and session-local span lookups.
|
|
2
|
+
ALTER TABLE herdr_panes ADD COLUMN agent TEXT;
|
|
3
|
+
ALTER TABLE herdr_panes ADD COLUMN last_seen_ms INTEGER;
|
|
4
|
+
UPDATE herdr_panes SET last_seen_ms = COALESCE(closed_ms, created_ms);
|
|
5
|
+
CREATE INDEX pane_sessions_session ON pane_sessions(session_id);
|
|
6
|
+
CREATE INDEX pane_sessions_pending ON pane_sessions(session_id) WHERE session_id IS NULL;
|
|
7
|
+
CREATE INDEX herdr_panes_cwd ON herdr_panes(cwd, created_ms);
|
|
8
|
+
CREATE INDEX herdr_panes_closed ON herdr_panes(closed_ms);
|
|
9
|
+
CREATE INDEX herdr_panes_seen ON herdr_panes(last_seen_ms);
|
|
10
|
+
CREATE INDEX sessions_cwd_started ON sessions(cwd, started_at);
|
|
11
|
+
CREATE INDEX otel_spans_conversation ON otel_spans(conversation_id);
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
-- Candidate membership is an index over portable events.data_json, not new captured text.
|
|
2
|
+
-- Backfill the one proven match on stores that resolved loads before this migration.
|
|
3
|
+
UPDATE events SET data_json = json_set(data_json, '$.attributionMatches', json_array(session_id))
|
|
4
|
+
WHERE source = 'mcp' AND kind = 'load' AND attribution_method = 'trace_handle'
|
|
5
|
+
AND json_extract(data_json, '$.attributionBasis') = 'tool_call_uri'
|
|
6
|
+
AND json_type(data_json, '$.attributionMatches') IS NULL;
|
|
7
|
+
|
|
8
|
+
CREATE INDEX events_pending_load_ts ON events(ts)
|
|
9
|
+
WHERE source = 'mcp' AND kind = 'load' AND session_id = 'unattributed' AND attribution_reason = 'ambiguous';
|
|
10
|
+
CREATE INDEX events_reconcile_load_uri ON events(runtime, skill_uri, ts)
|
|
11
|
+
WHERE source = 'mcp' AND kind = 'load';
|
|
12
|
+
CREATE INDEX events_uri_tools ON events(runtime, session_id, ts)
|
|
13
|
+
WHERE source = 'otel' AND kind = 'tool_call' AND session_id != 'unattributed'
|
|
14
|
+
AND json_extract(data_json, '$.success') IS NOT 0;
|
|
15
|
+
CREATE INDEX sessions_runtime_started ON sessions(runtime, started_at);
|
|
16
|
+
|
|
17
|
+
CREATE TABLE load_candidates (
|
|
18
|
+
load_id TEXT NOT NULL REFERENCES events(id) ON DELETE CASCADE,
|
|
19
|
+
session_id TEXT NOT NULL,
|
|
20
|
+
ts INTEGER NOT NULL,
|
|
21
|
+
PRIMARY KEY (load_id, session_id)
|
|
22
|
+
) WITHOUT ROWID;
|
|
23
|
+
CREATE INDEX load_candidates_session_ts ON load_candidates(session_id, ts, load_id);
|
|
24
|
+
|
|
25
|
+
-- Old history imports may have no candidate snapshot. Reconstruct at the event's timestamp,
|
|
26
|
+
-- exactly once on insert, not on every report. Both branches use session/time indexes.
|
|
27
|
+
CREATE VIEW load_candidate_rows AS
|
|
28
|
+
SELECT e.id AS load_id, j.value AS session_id, e.ts
|
|
29
|
+
FROM events e, json_each(COALESCE(json_extract(e.data_json, '$.attributionCandidates'),
|
|
30
|
+
(SELECT json_group_array(s.id) FROM sessions s
|
|
31
|
+
WHERE s.runtime = e.runtime AND s.started_at <= e.ts AND (s.ended_at IS NULL OR s.ended_at >= e.ts)
|
|
32
|
+
AND MAX(s.started_at, COALESCE((SELECT MAX(a.ts) FROM events a
|
|
33
|
+
WHERE a.session_id = s.id AND a.ts <= e.ts), s.started_at)) >= e.ts - 120000))) j
|
|
34
|
+
WHERE e.source = 'mcp' AND e.kind = 'load' AND j.type = 'text'
|
|
35
|
+
AND ((e.session_id = 'unattributed' AND e.attribution_reason = 'ambiguous')
|
|
36
|
+
OR (e.attribution_method = 'trace_handle' AND json_extract(e.data_json, '$.attributionBasis') = 'tool_call_uri'));
|
|
37
|
+
|
|
38
|
+
INSERT INTO load_candidates SELECT * FROM load_candidate_rows;
|
|
39
|
+
CREATE TRIGGER events_load_candidates_insert AFTER INSERT ON events
|
|
40
|
+
WHEN NEW.source = 'mcp' AND NEW.kind = 'load'
|
|
41
|
+
BEGIN
|
|
42
|
+
INSERT INTO load_candidates SELECT * FROM load_candidate_rows WHERE load_id = NEW.id;
|
|
43
|
+
END;
|
|
44
|
+
CREATE TRIGGER events_load_candidates_update AFTER UPDATE OF data_json ON events
|
|
45
|
+
WHEN NEW.source = 'mcp' AND NEW.kind = 'load'
|
|
46
|
+
AND json_extract(NEW.data_json, '$.attributionCandidates') IS NOT json_extract(OLD.data_json, '$.attributionCandidates')
|
|
47
|
+
BEGIN
|
|
48
|
+
DELETE FROM load_candidates WHERE load_id = NEW.id;
|
|
49
|
+
INSERT INTO load_candidates SELECT * FROM load_candidate_rows WHERE load_id = NEW.id;
|
|
50
|
+
END;
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
-- Identity snapshots, not SQLite rowids: history import and VACUUM may reorder rows.
|
|
2
|
+
-- NULL means this boundary predates recording; never reconstruct a historical cut.
|
|
3
|
+
ALTER TABLE flows ADD COLUMN start_event_ids_json TEXT;
|
|
4
|
+
ALTER TABLE flows ADD COLUMN end_event_ids_json TEXT;
|
|
5
|
+
ALTER TABLE flows ADD COLUMN end_sessions_json TEXT;
|
|
6
|
+
ALTER TABLE flows ADD COLUMN end_spans_json TEXT;
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
-- A recording domain names a database lineage, not an ordering of UUIDs.
|
|
2
|
+
-- Keep its high-water mark independently of retained observations and rowids.
|
|
3
|
+
CREATE TABLE recording_clock (
|
|
4
|
+
singleton INTEGER PRIMARY KEY CHECK (singleton = 1),
|
|
5
|
+
domain TEXT NOT NULL,
|
|
6
|
+
position INTEGER NOT NULL CHECK (position BETWEEN 0 AND 9007199254740991)
|
|
7
|
+
);
|
|
8
|
+
INSERT INTO recording_clock VALUES (1, lower(hex(randomblob(16))), 0);
|
|
9
|
+
-- Outer INSERT OR IGNORE must not suppress exhaustion and reuse the last mark.
|
|
10
|
+
CREATE TRIGGER recording_clock_range BEFORE UPDATE OF position ON recording_clock
|
|
11
|
+
WHEN typeof(NEW.position) <> 'integer' OR NEW.position < OLD.position OR NEW.position > 9007199254740991
|
|
12
|
+
BEGIN
|
|
13
|
+
SELECT RAISE(ABORT, 'recording clock exhausted or moved backwards');
|
|
14
|
+
END;
|
|
15
|
+
|
|
16
|
+
ALTER TABLE events ADD COLUMN recording_orders_json TEXT NOT NULL DEFAULT '{}';
|
|
17
|
+
ALTER TABLE otel_spans ADD COLUMN recording_orders_json TEXT NOT NULL DEFAULT '{}';
|
|
18
|
+
ALTER TABLE sessions ADD COLUMN end_orders_json TEXT NOT NULL DEFAULT '{}';
|
|
19
|
+
ALTER TABLE flows ADD COLUMN start_order_domain TEXT;
|
|
20
|
+
ALTER TABLE flows ADD COLUMN start_order INTEGER;
|
|
21
|
+
ALTER TABLE flows ADD COLUMN end_order_domain TEXT;
|
|
22
|
+
ALTER TABLE flows ADD COLUMN end_order INTEGER;
|
|
23
|
+
|
|
24
|
+
-- Old rows precede future cuts in this new domain. This does NOT reconstruct old
|
|
25
|
+
-- flow cuts or order the old rows against each other.
|
|
26
|
+
UPDATE events SET recording_orders_json = (SELECT json_object(domain, 0) FROM recording_clock);
|
|
27
|
+
UPDATE otel_spans SET recording_orders_json = (SELECT json_object(domain, 0) FROM recording_clock);
|
|
28
|
+
UPDATE sessions SET end_orders_json = (SELECT json_object(domain, 0) FROM recording_clock) WHERE ended_at IS NOT NULL;
|
|
29
|
+
|
|
30
|
+
-- Source-domain marks survive import; add local arrival order without replacing
|
|
31
|
+
-- a known mark when restoring history from this same lineage. Advance past it.
|
|
32
|
+
CREATE TRIGGER events_recording_insert AFTER INSERT ON events BEGIN
|
|
33
|
+
UPDATE recording_clock SET position = MAX(position + 1,
|
|
34
|
+
COALESCE(json_extract(NEW.recording_orders_json, '$."' || domain || '"'), 0));
|
|
35
|
+
UPDATE events SET recording_orders_json = (SELECT
|
|
36
|
+
json_insert(NEW.recording_orders_json, '$."' || domain || '"', position) FROM recording_clock)
|
|
37
|
+
WHERE id = NEW.id;
|
|
38
|
+
END;
|
|
39
|
+
|
|
40
|
+
CREATE TRIGGER spans_recording_insert AFTER INSERT ON otel_spans BEGIN
|
|
41
|
+
UPDATE recording_clock SET position = MAX(position + 1,
|
|
42
|
+
COALESCE(json_extract(NEW.recording_orders_json, '$."' || domain || '"'), 0));
|
|
43
|
+
UPDATE otel_spans SET recording_orders_json = (SELECT
|
|
44
|
+
json_insert(NEW.recording_orders_json, '$."' || domain || '"', position) FROM recording_clock)
|
|
45
|
+
WHERE trace_id = NEW.trace_id AND span_id = NEW.span_id;
|
|
46
|
+
END;
|
|
47
|
+
|
|
48
|
+
-- putSpans ignores replays. If stored facts are corrected, their previous order
|
|
49
|
+
-- cannot certify the corrected facts at a boundary that has already closed.
|
|
50
|
+
CREATE TRIGGER spans_recording_update AFTER UPDATE OF
|
|
51
|
+
parent_span_id, name, runtime, conversation_id, turn_key, call_id, start_ms, end_ms ON otel_spans
|
|
52
|
+
WHEN OLD.parent_span_id IS NOT NEW.parent_span_id OR OLD.name IS NOT NEW.name
|
|
53
|
+
OR OLD.runtime IS NOT NEW.runtime OR OLD.conversation_id IS NOT NEW.conversation_id
|
|
54
|
+
OR OLD.turn_key IS NOT NEW.turn_key OR OLD.call_id IS NOT NEW.call_id
|
|
55
|
+
OR OLD.start_ms IS NOT NEW.start_ms OR OLD.end_ms IS NOT NEW.end_ms
|
|
56
|
+
BEGIN
|
|
57
|
+
UPDATE recording_clock SET position = position + 1;
|
|
58
|
+
UPDATE otel_spans SET recording_orders_json = (SELECT json_object(domain, position) FROM recording_clock)
|
|
59
|
+
WHERE trace_id = NEW.trace_id AND span_id = NEW.span_id;
|
|
60
|
+
END;
|
|
61
|
+
|
|
62
|
+
CREATE TRIGGER sessions_end_insert AFTER INSERT ON sessions WHEN NEW.ended_at IS NOT NULL BEGIN
|
|
63
|
+
UPDATE recording_clock SET position = MAX(position + 1,
|
|
64
|
+
COALESCE(json_extract(NEW.end_orders_json, '$."' || domain || '"'), 0));
|
|
65
|
+
UPDATE sessions SET end_orders_json = (SELECT
|
|
66
|
+
json_insert(NEW.end_orders_json, '$."' || domain || '"', position) FROM recording_clock)
|
|
67
|
+
WHERE id = NEW.id;
|
|
68
|
+
END;
|
|
69
|
+
CREATE TRIGGER sessions_end_update AFTER UPDATE OF ended_at ON sessions WHEN OLD.ended_at IS NOT NEW.ended_at BEGIN
|
|
70
|
+
UPDATE recording_clock SET position = position + 1;
|
|
71
|
+
UPDATE sessions SET end_orders_json = (SELECT json_object(domain, position) FROM recording_clock) WHERE id = NEW.id;
|
|
72
|
+
END;
|