wendkeep 0.74.0 → 0.75.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/README.en.md +3 -3
- package/README.md +3 -3
- package/docs/en/commands/observer.md +22 -11
- package/docs/pt-BR/commands/observer.md +22 -11
- package/package.json +1 -1
- package/packages/cli/src/index.mjs +1 -1
- package/schema/observer/005-project-scoped-identities.sql +217 -0
- package/src/change.mjs +41 -1
- package/src/doctor.mjs +5 -0
- package/src/init.mjs +2 -2
- package/src/note.mjs +8 -1
- package/src/observer-publish.mjs +9 -34
- package/src/observer-server.mjs +38 -63
- package/src/observer-sql-migrate.mjs +1 -1
- package/src/observer-sql-publish.mjs +372 -12
- package/src/observer-sql-store.mjs +108 -28
- package/src/observer-store.mjs +15 -3
- package/src/observer.mjs +104 -14
|
@@ -9,7 +9,7 @@ import {
|
|
|
9
9
|
} from '../packages/vault/src/evidence-recall.mjs';
|
|
10
10
|
|
|
11
11
|
export const OBSERVER_SQL_FILE = 'observer.sqlite';
|
|
12
|
-
export const OBSERVER_SQL_SCHEMA_VERSION =
|
|
12
|
+
export const OBSERVER_SQL_SCHEMA_VERSION = 5;
|
|
13
13
|
export const OBSERVER_EVENT_SCHEMA_VERSION = 1;
|
|
14
14
|
|
|
15
15
|
const SCHEMA_DIR = fileURLToPath(new URL('../schema/observer/', import.meta.url));
|
|
@@ -22,6 +22,7 @@ const EVENT_KINDS = new Set([
|
|
|
22
22
|
const OBSERVER_SQL_MINIMUM_NODE = '22.13.0';
|
|
23
23
|
const require = createRequire(import.meta.url);
|
|
24
24
|
let DatabaseSync;
|
|
25
|
+
const DATABASE_PATHS = new WeakMap();
|
|
25
26
|
|
|
26
27
|
export function observerSqlRuntimeSupport(version = process.versions.node) {
|
|
27
28
|
const current = String(version || '0.0.0');
|
|
@@ -74,6 +75,10 @@ function hash(value) {
|
|
|
74
75
|
return createHash('sha256').update(typeof value === 'string' ? value : json(value)).digest('hex');
|
|
75
76
|
}
|
|
76
77
|
|
|
78
|
+
function scopedIdentity(projectId, externalId) {
|
|
79
|
+
return `${projectId}\u001f${externalId}`;
|
|
80
|
+
}
|
|
81
|
+
|
|
77
82
|
function validProjectId(value) { return typeof value === 'string' && PROJECT_ID_RE.test(value); }
|
|
78
83
|
|
|
79
84
|
function requireProject(db, projectId) {
|
|
@@ -97,7 +102,9 @@ export function openObserverDatabase(dataDir) {
|
|
|
97
102
|
if (!dataDir) throw new Error('dataDir é obrigatório.');
|
|
98
103
|
mkdirSync(dataDir, { recursive: true });
|
|
99
104
|
const SqliteDatabase = observerDatabaseSync();
|
|
100
|
-
const
|
|
105
|
+
const databasePath = join(dataDir, OBSERVER_SQL_FILE);
|
|
106
|
+
const db = new SqliteDatabase(databasePath);
|
|
107
|
+
DATABASE_PATHS.set(db, databasePath);
|
|
101
108
|
db.exec('PRAGMA foreign_keys = ON; PRAGMA journal_mode = WAL; PRAGMA busy_timeout = 5000;');
|
|
102
109
|
return db;
|
|
103
110
|
}
|
|
@@ -105,15 +112,39 @@ export function openObserverDatabase(dataDir) {
|
|
|
105
112
|
export function migrateObserverDatabase(db) {
|
|
106
113
|
if (!db) throw new Error('db é obrigatório.');
|
|
107
114
|
db.exec('CREATE TABLE IF NOT EXISTS schema_migrations (version INTEGER PRIMARY KEY, name TEXT NOT NULL, applied_at TEXT NOT NULL)');
|
|
108
|
-
const
|
|
115
|
+
const migrationColumns = db.prepare('PRAGMA table_info(schema_migrations)').all().map((row) => row.name);
|
|
116
|
+
if (!migrationColumns.includes('checksum')) db.exec("ALTER TABLE schema_migrations ADD COLUMN checksum TEXT NOT NULL DEFAULT ''");
|
|
117
|
+
const appliedRows = db.prepare('SELECT version, name, checksum FROM schema_migrations ORDER BY version').all();
|
|
118
|
+
const applied = new Map(appliedRows.map((row) => [Number(row.version), row]));
|
|
119
|
+
const backups = [];
|
|
109
120
|
for (const file of migrationFiles()) {
|
|
110
121
|
const version = Number(file.split('-')[0]);
|
|
111
|
-
if (applied.has(version)) continue;
|
|
112
122
|
const sql = readFileSync(join(SCHEMA_DIR, file), 'utf8');
|
|
123
|
+
const checksum = hash(sql);
|
|
124
|
+
const previous = applied.get(version);
|
|
125
|
+
if (previous) {
|
|
126
|
+
if (previous.name !== file || (previous.checksum && previous.checksum !== checksum)) {
|
|
127
|
+
const error = new Error(`Checksum da migração ${version} não corresponde ao arquivo ${file}.`);
|
|
128
|
+
error.code = 'WENDKEEP_OBSERVER_MIGRATION_CHECKSUM_MISMATCH';
|
|
129
|
+
throw error;
|
|
130
|
+
}
|
|
131
|
+
if (!previous.checksum) {
|
|
132
|
+
db.prepare('UPDATE schema_migrations SET checksum = ? WHERE version = ?').run(checksum, version);
|
|
133
|
+
}
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
if (/^\s*--\s*wendkeep:structural\b/m.test(sql) && applied.size > 0) {
|
|
137
|
+
const databasePath = DATABASE_PATHS.get(db);
|
|
138
|
+
if (!databasePath) throw new Error('Caminho do Observer desconhecido para backup estrutural.');
|
|
139
|
+
const backupPath = `${databasePath}.pre-${String(version).padStart(3, '0')}-${Date.now()}.bak`;
|
|
140
|
+
const escaped = backupPath.replaceAll("'", "''");
|
|
141
|
+
db.exec(`VACUUM INTO '${escaped}'`);
|
|
142
|
+
backups.push(backupPath);
|
|
143
|
+
}
|
|
113
144
|
db.exec('BEGIN IMMEDIATE');
|
|
114
145
|
try {
|
|
115
146
|
db.exec(sql);
|
|
116
|
-
db.prepare('INSERT INTO schema_migrations(version, name, applied_at) VALUES (?, ?, ?)').run(version, file, now());
|
|
147
|
+
db.prepare('INSERT INTO schema_migrations(version, name, applied_at, checksum) VALUES (?, ?, ?, ?)').run(version, file, now(), checksum);
|
|
117
148
|
db.exec('COMMIT');
|
|
118
149
|
} catch (error) {
|
|
119
150
|
db.exec('ROLLBACK');
|
|
@@ -123,7 +154,8 @@ export function migrateObserverDatabase(db) {
|
|
|
123
154
|
rebuildSqlEvidenceIndex(db, { missingOnly: true });
|
|
124
155
|
return {
|
|
125
156
|
version: Number(db.prepare('SELECT COALESCE(MAX(version), 0) AS version FROM schema_migrations').get().version || 0),
|
|
126
|
-
applied: db.prepare('SELECT version, name, applied_at FROM schema_migrations ORDER BY version').all(),
|
|
157
|
+
applied: db.prepare('SELECT version, name, applied_at, checksum FROM schema_migrations ORDER BY version').all(),
|
|
158
|
+
backups,
|
|
127
159
|
};
|
|
128
160
|
}
|
|
129
161
|
|
|
@@ -268,9 +300,9 @@ function ensureSession(db, projectId, payload) {
|
|
|
268
300
|
const sessionId = text(payload.session_id || payload.sessionId);
|
|
269
301
|
if (!sessionId) throw new Error('session_id ausente.');
|
|
270
302
|
db.prepare(`
|
|
271
|
-
INSERT INTO sessions(session_id, project_id, provider, status, summary, change_slug, started_at, ended_at, updated_at, metadata_json)
|
|
272
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
273
|
-
ON CONFLICT(session_id) DO UPDATE SET
|
|
303
|
+
INSERT INTO sessions(session_pk, session_id, project_id, provider, status, summary, change_slug, started_at, ended_at, updated_at, metadata_json)
|
|
304
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
305
|
+
ON CONFLICT(project_id, session_id) DO UPDATE SET
|
|
274
306
|
provider = CASE WHEN excluded.provider <> '' THEN excluded.provider ELSE sessions.provider END,
|
|
275
307
|
status = CASE WHEN excluded.status <> '' THEN excluded.status ELSE sessions.status END,
|
|
276
308
|
summary = CASE WHEN excluded.summary <> '' THEN excluded.summary ELSE sessions.summary END,
|
|
@@ -280,7 +312,7 @@ function ensureSession(db, projectId, payload) {
|
|
|
280
312
|
updated_at = excluded.updated_at,
|
|
281
313
|
metadata_json = CASE WHEN excluded.metadata_json <> '{}' THEN excluded.metadata_json ELSE sessions.metadata_json END
|
|
282
314
|
`).run(
|
|
283
|
-
sessionId, projectId, text(payload.provider), text(payload.status, 'unknown'), text(payload.summary), text(payload.change_slug || payload.changeSlug),
|
|
315
|
+
scopedIdentity(projectId, sessionId), sessionId, projectId, text(payload.provider), text(payload.status, 'unknown'), text(payload.summary), text(payload.change_slug || payload.changeSlug),
|
|
284
316
|
payload.started_at || payload.startedAt || null, payload.ended_at || payload.endedAt || null, now(), json(payload.metadata),
|
|
285
317
|
);
|
|
286
318
|
return sessionId;
|
|
@@ -291,9 +323,9 @@ function ensureAgent(db, projectId, payload) {
|
|
|
291
323
|
if (!agentId) throw new Error('agent_id ausente.');
|
|
292
324
|
const sessionId = ensureSession(db, projectId, payload);
|
|
293
325
|
db.prepare(`
|
|
294
|
-
INSERT INTO agent_runs(agent_id, project_id, session_id, parent_agent_id, role, agent_name, agent_type, workflow, status, model, effort, started_at, ended_at, metadata_json)
|
|
295
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
296
|
-
ON CONFLICT(agent_id) DO UPDATE SET
|
|
326
|
+
INSERT INTO agent_runs(agent_pk, agent_id, project_id, session_id, parent_agent_id, role, agent_name, agent_type, workflow, status, model, effort, started_at, ended_at, metadata_json)
|
|
327
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
328
|
+
ON CONFLICT(project_id, agent_id) DO UPDATE SET
|
|
297
329
|
parent_agent_id = COALESCE(excluded.parent_agent_id, agent_runs.parent_agent_id),
|
|
298
330
|
role = excluded.role,
|
|
299
331
|
agent_name = CASE WHEN excluded.agent_name <> '' THEN excluded.agent_name ELSE agent_runs.agent_name END,
|
|
@@ -306,7 +338,7 @@ function ensureAgent(db, projectId, payload) {
|
|
|
306
338
|
ended_at = COALESCE(excluded.ended_at, agent_runs.ended_at),
|
|
307
339
|
metadata_json = CASE WHEN excluded.metadata_json <> '{}' THEN excluded.metadata_json ELSE agent_runs.metadata_json END
|
|
308
340
|
`).run(
|
|
309
|
-
agentId, projectId, sessionId, payload.parent_agent_id || payload.parentAgentId || null, text(payload.role, 'main'),
|
|
341
|
+
scopedIdentity(projectId, agentId), agentId, projectId, sessionId, payload.parent_agent_id || payload.parentAgentId || null, text(payload.role, 'main'),
|
|
310
342
|
text(payload.agent_name || payload.agentName), text(payload.agent_type || payload.agentType), text(payload.workflow),
|
|
311
343
|
text(payload.status, 'unknown'), text(payload.model), text(payload.effort), payload.started_at || null, payload.ended_at || null, json(payload.metadata),
|
|
312
344
|
);
|
|
@@ -398,19 +430,19 @@ function applyUsageRollup(db, event) {
|
|
|
398
430
|
const [input, cacheWrite, cacheRead, output, reasoning, total] = tokenFields(p.tokens);
|
|
399
431
|
const rollupKey = text(p.rollup_key || p.rollupKey) || [event.project_id, sessionId, agentId, p.model_provider || p.modelProvider || '', p.model || '', p.effort || ''].join(':');
|
|
400
432
|
const revision = integer(p.revision, 1);
|
|
401
|
-
const current = db.prepare('SELECT revision FROM usage_rollups WHERE rollup_key = ?').get(rollupKey);
|
|
433
|
+
const current = db.prepare('SELECT revision FROM usage_rollups WHERE project_id = ? AND rollup_key = ?').get(event.project_id, rollupKey);
|
|
402
434
|
if (current && revision < Number(current.revision)) return { stale: true };
|
|
403
435
|
db.prepare(`
|
|
404
|
-
INSERT INTO usage_rollups(rollup_key, project_id, session_id, agent_id, role, provider, model_provider, model, effort, calls, tokens_input, tokens_cache_write, tokens_cache_read, tokens_output, tokens_reasoning, tokens_total, cost_usd, cost_status, pricing_source, pricing_version, wasted_usd, revision, occurred_at, source_event_id, metadata_json)
|
|
405
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
406
|
-
ON CONFLICT(rollup_key) DO UPDATE SET
|
|
436
|
+
INSERT INTO usage_rollups(rollup_pk, rollup_key, project_id, session_id, agent_id, role, provider, model_provider, model, effort, calls, tokens_input, tokens_cache_write, tokens_cache_read, tokens_output, tokens_reasoning, tokens_total, cost_usd, cost_status, pricing_source, pricing_version, wasted_usd, revision, occurred_at, source_event_id, metadata_json)
|
|
437
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
438
|
+
ON CONFLICT(project_id, rollup_key) DO UPDATE SET
|
|
407
439
|
calls = excluded.calls, tokens_input = excluded.tokens_input, tokens_cache_write = excluded.tokens_cache_write,
|
|
408
440
|
tokens_cache_read = excluded.tokens_cache_read, tokens_output = excluded.tokens_output, tokens_reasoning = excluded.tokens_reasoning,
|
|
409
441
|
tokens_total = excluded.tokens_total, cost_usd = excluded.cost_usd, cost_status = excluded.cost_status,
|
|
410
442
|
pricing_source = excluded.pricing_source, pricing_version = excluded.pricing_version, wasted_usd = excluded.wasted_usd,
|
|
411
443
|
revision = excluded.revision, occurred_at = excluded.occurred_at, source_event_id = excluded.source_event_id, metadata_json = excluded.metadata_json
|
|
412
444
|
`).run(
|
|
413
|
-
rollupKey, event.project_id, sessionId, agentId, text(p.role, 'main'), text(p.provider), text(p.model_provider || p.modelProvider), text(p.model), text(p.effort),
|
|
445
|
+
scopedIdentity(event.project_id, rollupKey), rollupKey, event.project_id, sessionId, agentId, text(p.role, 'main'), text(p.provider), text(p.model_provider || p.modelProvider), text(p.model), text(p.effort),
|
|
414
446
|
integer(p.calls), input, cacheWrite, cacheRead, output, reasoning, total, number(p.cost_usd ?? p.costUsd), text(p.cost_status || p.costStatus, 'unknown'),
|
|
415
447
|
text(p.pricing_source || p.pricingSource), text(p.pricing_version || p.pricingVersion), number(p.wasted_usd ?? p.wastedUsd), revision, event.occurred_at, event.event_id, json(p.metadata),
|
|
416
448
|
);
|
|
@@ -423,17 +455,17 @@ function applyCall(db, event) {
|
|
|
423
455
|
const agentId = ensureAgent(db, event.project_id, p);
|
|
424
456
|
const callId = text(p.call_id || p.callId);
|
|
425
457
|
if (!callId) throw new Error('call_id ausente.');
|
|
426
|
-
if (db.prepare('SELECT call_id FROM llm_calls WHERE call_id = ?').get(callId)) {
|
|
458
|
+
if (db.prepare('SELECT call_id FROM llm_calls WHERE project_id = ? AND call_id = ?').get(event.project_id, callId)) {
|
|
427
459
|
const error = new Error(`call_id já existe: ${callId}`);
|
|
428
460
|
error.code = 'call_conflict';
|
|
429
461
|
throw error;
|
|
430
462
|
}
|
|
431
463
|
const [input, cacheWrite, cacheRead, output, reasoning, total] = tokenFields(p.tokens);
|
|
432
464
|
db.prepare(`
|
|
433
|
-
INSERT INTO llm_calls(call_id, project_id, session_id, agent_id, role, provider, model_provider, model, effort, sequence, occurred_at, tokens_input, tokens_cache_write, tokens_cache_read, tokens_output, tokens_reasoning, tokens_total, cost_usd, cost_status, transcript_id, prompt_text, response_text, status, metadata_json)
|
|
434
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
465
|
+
INSERT INTO llm_calls(call_pk, call_id, project_id, session_id, agent_id, role, provider, model_provider, model, effort, sequence, occurred_at, tokens_input, tokens_cache_write, tokens_cache_read, tokens_output, tokens_reasoning, tokens_total, cost_usd, cost_status, transcript_id, prompt_text, response_text, status, metadata_json)
|
|
466
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
435
467
|
`).run(
|
|
436
|
-
callId, event.project_id, sessionId, agentId, text(p.role, 'main'), text(p.provider), text(p.model_provider || p.modelProvider), text(p.model), text(p.effort), integer(p.sequence),
|
|
468
|
+
scopedIdentity(event.project_id, callId), callId, event.project_id, sessionId, agentId, text(p.role, 'main'), text(p.provider), text(p.model_provider || p.modelProvider), text(p.model), text(p.effort), integer(p.sequence),
|
|
437
469
|
text(p.occurred_at || event.occurred_at), input, cacheWrite, cacheRead, output, reasoning, total, number(p.cost_usd ?? p.costUsd), text(p.cost_status || p.costStatus, 'unknown'),
|
|
438
470
|
p.transcript_id || p.transcriptId || null, text(p.prompt_text || p.promptText || p.prompt), text(p.response_text || p.responseText || p.response), text(p.status, 'complete'), json(p.metadata),
|
|
439
471
|
);
|
|
@@ -448,14 +480,14 @@ function applyTranscript(db, event) {
|
|
|
448
480
|
if (!transcriptId) throw new Error('transcript_id ausente.');
|
|
449
481
|
const encoded = encodeTranscript(p.content);
|
|
450
482
|
db.prepare(`
|
|
451
|
-
INSERT INTO transcripts(transcript_id, project_id, session_id, agent_id, coverage, codec, content_gzip, content_sha256, original_bytes, compressed_bytes, source, occurred_at, metadata_json)
|
|
452
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
453
|
-
ON CONFLICT(transcript_id) DO UPDATE SET
|
|
483
|
+
INSERT INTO transcripts(transcript_pk, transcript_id, project_id, session_id, agent_id, coverage, codec, content_gzip, content_sha256, original_bytes, compressed_bytes, source, occurred_at, metadata_json)
|
|
484
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
485
|
+
ON CONFLICT(project_id, transcript_id) DO UPDATE SET
|
|
454
486
|
coverage = excluded.coverage, codec = excluded.codec, content_gzip = excluded.content_gzip,
|
|
455
487
|
content_sha256 = excluded.content_sha256, original_bytes = excluded.original_bytes, compressed_bytes = excluded.compressed_bytes,
|
|
456
488
|
source = excluded.source, occurred_at = excluded.occurred_at, metadata_json = excluded.metadata_json
|
|
457
489
|
`).run(
|
|
458
|
-
transcriptId, event.project_id, sessionId, agentId, text(p.coverage, 'complete'), encoded.codec, encoded.content_gzip, encoded.content_sha256,
|
|
490
|
+
scopedIdentity(event.project_id, transcriptId), transcriptId, event.project_id, sessionId, agentId, text(p.coverage, 'complete'), encoded.codec, encoded.content_gzip, encoded.content_sha256,
|
|
459
491
|
encoded.original_bytes, encoded.compressed_bytes, text(p.source), event.occurred_at, json(p.metadata),
|
|
460
492
|
);
|
|
461
493
|
return { stale: false };
|
|
@@ -501,6 +533,8 @@ export function ingestObserverEvents(db, { projectId, events = [] } = {}) {
|
|
|
501
533
|
}
|
|
502
534
|
continue;
|
|
503
535
|
}
|
|
536
|
+
db.exec('SAVEPOINT ingest_one');
|
|
537
|
+
let savepointOpen = true;
|
|
504
538
|
try {
|
|
505
539
|
db.prepare('INSERT INTO ingest_events(event_id, project_id, kind, payload_hash, payload_json, occurred_at, ingested_at, status) VALUES (?, ?, ?, ?, ?, ?, ?, ?)')
|
|
506
540
|
.run(event.event_id, projectId, event.kind, payloadHash, json(event.payload), event.occurred_at, now(), 'accepted');
|
|
@@ -513,8 +547,12 @@ export function ingestObserverEvents(db, { projectId, events = [] } = {}) {
|
|
|
513
547
|
result.accepted += 1;
|
|
514
548
|
result.results.push({ accepted: true, event_id: event.event_id });
|
|
515
549
|
}
|
|
550
|
+
db.exec('RELEASE ingest_one');
|
|
551
|
+
savepointOpen = false;
|
|
516
552
|
} catch (error) {
|
|
517
|
-
|
|
553
|
+
if (savepointOpen) {
|
|
554
|
+
try { db.exec('ROLLBACK TO ingest_one'); } finally { db.exec('RELEASE ingest_one'); }
|
|
555
|
+
}
|
|
518
556
|
if (error?.code === 'call_conflict' || error?.code === 'document_conflict') {
|
|
519
557
|
result.conflicts += 1;
|
|
520
558
|
result.results.push({ accepted: false, conflict: true, event_id: event.event_id, errors: [error.message] });
|
|
@@ -704,6 +742,48 @@ export function readSqlProject(db, projectId) {
|
|
|
704
742
|
return db.prepare('SELECT * FROM projects WHERE project_id = ?').get(projectId);
|
|
705
743
|
}
|
|
706
744
|
|
|
745
|
+
export function upsertSqlProjectSnapshot(db, snapshot) {
|
|
746
|
+
const projectId = text(snapshot?.project_id || snapshot?.projectId);
|
|
747
|
+
requireProject(db, projectId);
|
|
748
|
+
const eventId = text(snapshot?.event_id);
|
|
749
|
+
const capturedAt = text(snapshot?.captured_at);
|
|
750
|
+
if (!eventId || !capturedAt || Number.isNaN(Date.parse(capturedAt))) throw new Error('snapshot SQL inválido.');
|
|
751
|
+
const current = db.prepare('SELECT event_id, captured_at FROM project_snapshots WHERE project_id = ?').get(projectId);
|
|
752
|
+
if (current?.event_id === eventId) return { accepted: false, duplicate: true, event_id: eventId };
|
|
753
|
+
if (current && (current.captured_at > capturedAt
|
|
754
|
+
|| (current.captured_at === capturedAt && current.event_id >= eventId))) {
|
|
755
|
+
return { accepted: false, stale: true, event_id: eventId };
|
|
756
|
+
}
|
|
757
|
+
db.prepare(`INSERT INTO project_snapshots(project_id, event_id, captured_at, snapshot_json)
|
|
758
|
+
VALUES (?, ?, ?, ?)
|
|
759
|
+
ON CONFLICT(project_id) DO UPDATE SET
|
|
760
|
+
event_id = excluded.event_id,
|
|
761
|
+
captured_at = excluded.captured_at,
|
|
762
|
+
snapshot_json = excluded.snapshot_json`).run(projectId, eventId, capturedAt, json(snapshot));
|
|
763
|
+
return { accepted: true, duplicate: false, event_id: eventId };
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
export function readSqlProjectSnapshot(db, projectId) {
|
|
767
|
+
requireProject(db, projectId);
|
|
768
|
+
const row = db.prepare('SELECT snapshot_json FROM project_snapshots WHERE project_id = ?').get(projectId);
|
|
769
|
+
return row ? parseJson(row.snapshot_json, null) : null;
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
export function readSqlProjectOverview(db, projectId) {
|
|
773
|
+
const project = readSqlProject(db, projectId);
|
|
774
|
+
const snapshot = readSqlProjectSnapshot(db, projectId);
|
|
775
|
+
const events = db.prepare('SELECT COUNT(*) AS count FROM ingest_events WHERE project_id = ?').get(projectId);
|
|
776
|
+
return {
|
|
777
|
+
projectId: project.project_id,
|
|
778
|
+
projectName: project.project_name,
|
|
779
|
+
wendkeepVersion: project.wendkeep_version,
|
|
780
|
+
registeredAt: project.registered_at,
|
|
781
|
+
updatedAt: project.updated_at,
|
|
782
|
+
eventCount: integer(events.count),
|
|
783
|
+
snapshot,
|
|
784
|
+
};
|
|
785
|
+
}
|
|
786
|
+
|
|
707
787
|
export function listSqlProjects(db) {
|
|
708
788
|
return db.prepare('SELECT * FROM projects ORDER BY project_name, project_id').all();
|
|
709
789
|
}
|
package/src/observer-store.mjs
CHANGED
|
@@ -86,8 +86,7 @@ function newer(left, right) {
|
|
|
86
86
|
return leftTime > rightTime || (leftTime === rightTime && String(left?.event_id).localeCompare(String(right?.event_id)) > 0);
|
|
87
87
|
}
|
|
88
88
|
|
|
89
|
-
|
|
90
|
-
ensureDataDir(dataDir);
|
|
89
|
+
function deriveObserverIndex(dataDir) {
|
|
91
90
|
const byProject = new Map();
|
|
92
91
|
for (const event of readEvents(dataDir)) {
|
|
93
92
|
const current = byProject.get(event.project_id);
|
|
@@ -106,15 +105,28 @@ export function rebuildObserverIndex(dataDir) {
|
|
|
106
105
|
const item = byProject.get(event.project_id);
|
|
107
106
|
if (item) item.eventCount += 1;
|
|
108
107
|
}
|
|
109
|
-
|
|
108
|
+
return {
|
|
110
109
|
schema_version: OBSERVER_DATA_SCHEMA_VERSION,
|
|
111
110
|
generated_at: new Date().toISOString(),
|
|
112
111
|
projects: [...byProject.values()].sort((a, b) => a.projectId.localeCompare(b.projectId)),
|
|
113
112
|
};
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export function rebuildObserverIndex(dataDir) {
|
|
116
|
+
ensureDataDir(dataDir);
|
|
117
|
+
const index = deriveObserverIndex(dataDir);
|
|
114
118
|
atomicJson(join(dataDir, OBSERVER_INDEX_FILE), index);
|
|
115
119
|
return index;
|
|
116
120
|
}
|
|
117
121
|
|
|
122
|
+
/** Read a legacy source without creating or refreshing legacy authority files. */
|
|
123
|
+
export function readObserverIndexSource(dataDir) {
|
|
124
|
+
ensureDataDir(dataDir);
|
|
125
|
+
const index = readJson(join(dataDir, OBSERVER_INDEX_FILE), null);
|
|
126
|
+
if (index?.schema_version === OBSERVER_DATA_SCHEMA_VERSION && Array.isArray(index.projects)) return index;
|
|
127
|
+
return deriveObserverIndex(dataDir);
|
|
128
|
+
}
|
|
129
|
+
|
|
118
130
|
export function readObserverIndex(dataDir) {
|
|
119
131
|
ensureDataDir(dataDir);
|
|
120
132
|
const path = join(dataDir, OBSERVER_INDEX_FILE);
|
package/src/observer.mjs
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { homedir } from 'node:os';
|
|
2
2
|
import { isAbsolute, resolve } from 'node:path';
|
|
3
|
-
import {
|
|
3
|
+
import { readObserverIndexSource } from './observer-store.mjs';
|
|
4
4
|
import { buildProjectSnapshot } from './observer-snapshot.mjs';
|
|
5
5
|
import { compareMemoryParity } from './observer-memory-publish.mjs';
|
|
6
6
|
import { publishObserverSql } from './observer-sql-publish.mjs';
|
|
7
|
+
import { migrateObserverData } from './observer-sql-migrate.mjs';
|
|
7
8
|
import { startObserverServer } from './observer-server.mjs';
|
|
8
|
-
import { ensureObserverDatabase, migrateObserverDatabase, listSqlProjects, OBSERVER_SQL_FILE, OBSERVER_SQL_SCHEMA_VERSION } from './observer-sql-store.mjs';
|
|
9
|
+
import { ensureObserverDatabase, migrateObserverDatabase, listSqlProjects, registerSqlProject, upsertSqlProjectSnapshot, OBSERVER_SQL_FILE, OBSERVER_SQL_SCHEMA_VERSION } from './observer-sql-store.mjs';
|
|
9
10
|
import { resolveProjectVault } from '../packages/vault/src/project-vault.mjs';
|
|
10
11
|
import { observerAuthHeaders, resolveObserverToken } from './observer-auth.mjs';
|
|
11
12
|
|
|
@@ -16,12 +17,15 @@ Uso:
|
|
|
16
17
|
[--allow-non-loopback] [--token TOKEN]
|
|
17
18
|
wendkeep observer register --project P [--vault V] [--data-dir D] [--json]
|
|
18
19
|
wendkeep observer publish --project P [--vault V] [--data-dir D] [--json]
|
|
20
|
+
wendkeep observer reconcile --project P [--vault V] [--data-dir D] [--url U]
|
|
21
|
+
[--capture-level metadata|messages|full-transcript] [--json]
|
|
19
22
|
wendkeep observer memory import --project P [--vault V] [--url U] [--token TOKEN]
|
|
20
23
|
[--capture-level metadata|messages|full-transcript] [--json]
|
|
21
24
|
wendkeep observer status [--data-dir D] [--json]
|
|
22
25
|
|
|
23
26
|
O Observer local pode manter snapshots operacionais e uma cópia completa da memória em volume
|
|
24
27
|
Docker. O comando memory import faz a primeira migração de um vault para o container.
|
|
28
|
+
Hooks apenas enfileiram/drenam alterações incrementais; reconcile é a varredura integral explícita.
|
|
25
29
|
`;
|
|
26
30
|
|
|
27
31
|
function optionValue(argv, name) {
|
|
@@ -73,6 +77,19 @@ function databaseSummary(dir) {
|
|
|
73
77
|
} finally { db.close(); }
|
|
74
78
|
}
|
|
75
79
|
|
|
80
|
+
function sqlProjectsSummary(dir) {
|
|
81
|
+
const db = ensureObserverDatabase(dir);
|
|
82
|
+
try {
|
|
83
|
+
return listSqlProjects(db).map((project) => ({
|
|
84
|
+
projectId: project.project_id,
|
|
85
|
+
projectName: project.project_name,
|
|
86
|
+
wendkeepVersion: project.wendkeep_version,
|
|
87
|
+
updatedAt: project.updated_at,
|
|
88
|
+
authority: 'sqlite',
|
|
89
|
+
}));
|
|
90
|
+
} finally { db.close(); }
|
|
91
|
+
}
|
|
92
|
+
|
|
76
93
|
export async function runObserver(argv = [], { write = (chunk) => process.stdout.write(chunk) } = {}) {
|
|
77
94
|
const [sub] = argv;
|
|
78
95
|
const asJson = argv.includes('--json');
|
|
@@ -84,7 +101,8 @@ export async function runObserver(argv = [], { write = (chunk) => process.stdout
|
|
|
84
101
|
const token = resolveObserverToken(optionValue(argv, '--token'));
|
|
85
102
|
|
|
86
103
|
if (sub === 'status') {
|
|
87
|
-
|
|
104
|
+
const legacy = summary(readObserverIndexSource(dir));
|
|
105
|
+
print({ ...legacy, projects: sqlProjectsSummary(dir), legacy_projects: legacy.projects, database: databaseSummary(dir) }, asJson, write);
|
|
88
106
|
return 0;
|
|
89
107
|
}
|
|
90
108
|
|
|
@@ -131,6 +149,12 @@ export async function runObserver(argv = [], { write = (chunk) => process.stdout
|
|
|
131
149
|
token,
|
|
132
150
|
captureLevel: optionValue(argv, '--capture-level') || process.env.WENDKEEP_OBSERVER_CAPTURE_LEVEL || 'metadata',
|
|
133
151
|
});
|
|
152
|
+
const snapshotResponse = await fetch(`${String(url).replace(/\/$/, '')}/v1/projects/${encodeURIComponent(snapshot.project_id)}/snapshot`, {
|
|
153
|
+
method: 'POST',
|
|
154
|
+
headers,
|
|
155
|
+
body: JSON.stringify(snapshot),
|
|
156
|
+
});
|
|
157
|
+
if (!snapshotResponse.ok) throw new Error(`Observer não importou o snapshot: HTTP ${snapshotResponse.status}.`);
|
|
134
158
|
const parity = await compareMemoryParity({
|
|
135
159
|
vaultBase: vault,
|
|
136
160
|
projectId: snapshot.project_id,
|
|
@@ -148,24 +172,90 @@ export async function runObserver(argv = [], { write = (chunk) => process.stdout
|
|
|
148
172
|
return result.ok ? 0 : 1;
|
|
149
173
|
}
|
|
150
174
|
|
|
175
|
+
if (sub === 'reconcile') {
|
|
176
|
+
const root = projectRoot(argv);
|
|
177
|
+
const vault = vaultBase(argv, root);
|
|
178
|
+
const snapshot = buildProjectSnapshot({ vaultBase: vault, projectRoot: root });
|
|
179
|
+
const url = optionValue(argv, '--url') || process.env.WENDKEEP_OBSERVER_URL || '';
|
|
180
|
+
if (!url) {
|
|
181
|
+
const db = ensureObserverDatabase(dir);
|
|
182
|
+
let migration;
|
|
183
|
+
try {
|
|
184
|
+
migration = migrateObserverData({
|
|
185
|
+
dataDir: dir,
|
|
186
|
+
vaultBase: vault,
|
|
187
|
+
projectId: snapshot.project_id,
|
|
188
|
+
projectName: snapshot.project_name,
|
|
189
|
+
database: db,
|
|
190
|
+
});
|
|
191
|
+
upsertSqlProjectSnapshot(db, snapshot);
|
|
192
|
+
} finally { db.close(); }
|
|
193
|
+
const result = { ok: migration.rejected === 0 && migration.conflicts === 0, project_id: snapshot.project_id, mode: 'local-sqlite', migration };
|
|
194
|
+
print(result, asJson, write);
|
|
195
|
+
return result.ok ? 0 : 1;
|
|
196
|
+
}
|
|
197
|
+
const registration = await fetch(`${String(url).replace(/\/$/, '')}/v1/projects/${encodeURIComponent(snapshot.project_id)}`, {
|
|
198
|
+
method: 'PUT',
|
|
199
|
+
headers: observerAuthHeaders(token, { 'content-type': 'application/json', accept: 'application/json' }),
|
|
200
|
+
body: JSON.stringify({
|
|
201
|
+
project_id: snapshot.project_id,
|
|
202
|
+
project_name: snapshot.project_name,
|
|
203
|
+
wendkeep_version: snapshot.wendkeep_version,
|
|
204
|
+
}),
|
|
205
|
+
});
|
|
206
|
+
if (!registration.ok) throw new Error(`Observer não registrou o projeto: HTTP ${registration.status}.`);
|
|
207
|
+
const sql = await publishObserverSql({
|
|
208
|
+
vaultBase: vault,
|
|
209
|
+
projectId: snapshot.project_id,
|
|
210
|
+
url,
|
|
211
|
+
token,
|
|
212
|
+
captureLevel: optionValue(argv, '--capture-level') || process.env.WENDKEEP_OBSERVER_CAPTURE_LEVEL || 'metadata',
|
|
213
|
+
});
|
|
214
|
+
const snapshotResponse = await fetch(`${String(url).replace(/\/$/, '')}/v1/projects/${encodeURIComponent(snapshot.project_id)}/snapshot`, {
|
|
215
|
+
method: 'POST',
|
|
216
|
+
headers: observerAuthHeaders(token, { 'content-type': 'application/json', accept: 'application/json' }),
|
|
217
|
+
body: JSON.stringify(snapshot),
|
|
218
|
+
});
|
|
219
|
+
if (!snapshotResponse.ok) throw new Error(`Observer não reconciliou o snapshot: HTTP ${snapshotResponse.status}.`);
|
|
220
|
+
const parity = await compareMemoryParity({ vaultBase: vault, projectId: snapshot.project_id, url, token });
|
|
221
|
+
const result = { ok: sql.ok && parity.missing === 0 && parity.mismatched === 0, project_id: snapshot.project_id, mode: 'remote-sqlite', sql, parity };
|
|
222
|
+
print(result, asJson, write);
|
|
223
|
+
return result.ok ? 0 : 1;
|
|
224
|
+
}
|
|
225
|
+
|
|
151
226
|
if (!['register', 'publish'].includes(sub)) throw new Error('observer: subcomando desconhecido: ' + sub);
|
|
152
227
|
const root = projectRoot(argv);
|
|
153
228
|
const vault = vaultBase(argv, root);
|
|
154
229
|
const snapshot = buildProjectSnapshot({ vaultBase: vault, projectRoot: root });
|
|
155
230
|
|
|
156
231
|
if (sub === 'register') {
|
|
157
|
-
const
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
232
|
+
const db = ensureObserverDatabase(dir);
|
|
233
|
+
let sql;
|
|
234
|
+
try {
|
|
235
|
+
sql = registerSqlProject(db, {
|
|
236
|
+
projectId: snapshot.project_id,
|
|
237
|
+
projectName: snapshot.project_name,
|
|
238
|
+
wendkeepVersion: snapshot.wendkeep_version,
|
|
239
|
+
});
|
|
240
|
+
} finally { db.close(); }
|
|
241
|
+
if (!sql.registered) throw new Error(sql.errors.join(' '));
|
|
242
|
+
print({ ...sql, authority: 'sqlite' }, asJson, write);
|
|
164
243
|
return 0;
|
|
165
244
|
}
|
|
166
245
|
|
|
167
|
-
const
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
246
|
+
const db = ensureObserverDatabase(dir);
|
|
247
|
+
let migration;
|
|
248
|
+
try {
|
|
249
|
+
migration = migrateObserverData({
|
|
250
|
+
dataDir: dir,
|
|
251
|
+
vaultBase: vault,
|
|
252
|
+
projectId: snapshot.project_id,
|
|
253
|
+
projectName: snapshot.project_name,
|
|
254
|
+
database: db,
|
|
255
|
+
});
|
|
256
|
+
upsertSqlProjectSnapshot(db, snapshot);
|
|
257
|
+
} finally { db.close(); }
|
|
258
|
+
const result = { ok: migration.rejected === 0 && migration.conflicts === 0, authority: 'sqlite', migration };
|
|
259
|
+
print(result, asJson, write);
|
|
260
|
+
return result.ok ? 0 : 1;
|
|
171
261
|
}
|