skillar 0.2.1 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +280 -0
- package/README.md +37 -5
- package/dist/analysis/finding.js +8 -190
- package/dist/analysis/kernel.js +896 -0
- package/dist/analysis/tree.js +7 -154
- package/dist/analysis/usage.js +9 -394
- package/dist/api/flows.js +42 -212
- package/dist/api/report-kernel.js +1063 -0
- package/dist/api/routes.js +12 -2
- package/dist/catalog/fields.js +138 -146
- package/dist/catalog/watcher.js +7 -2
- package/dist/cli/init.js +45 -0
- package/dist/cli/main.js +76 -3
- package/dist/cli/private-skills.js +122 -0
- package/dist/cli/serve.js +15 -1
- package/dist/cli/sync.js +124 -0
- package/dist/dashboard/NOTICE +280 -0
- package/dist/dashboard/assets/app.js +737 -275
- package/dist/dashboard/assets/astryx-reset.css +407 -0
- package/dist/dashboard/assets/astryx.css +1077 -0
- package/dist/dashboard/assets/charts.css +54 -0
- package/dist/dashboard/assets/chunk-TPGDA2J4.js +1 -0
- package/dist/dashboard/assets/chunk-YLWHVDMP.js +36 -0
- package/dist/dashboard/assets/controls.css +207 -0
- package/dist/dashboard/assets/overlays.css +132 -0
- package/dist/dashboard/assets/p1.css +135 -0
- package/dist/dashboard/assets/shell.css +71 -0
- package/dist/dashboard/assets/skillar-theme.css +692 -0
- package/dist/dashboard/assets/styles.css +224 -121
- package/dist/dashboard/assets/tokens.css +278 -0
- package/dist/dashboard/assets/widgets.css +83 -0
- package/dist/dashboard/index.html +11 -2
- package/dist/dashboard/viewers/viewers.css +61 -0
- package/dist/ingest/attribution.js +5 -1
- package/dist/ingest/pipeline.js +2 -0
- package/dist/ingest/request-recorder.js +39 -0
- package/dist/mcp/dispatch.js +2 -2
- package/dist/mcp/http.js +43 -3
- package/dist/mcp/skills-extension.js +42 -15
- package/dist/mcp/stdio-proxy.js +24 -3
- package/dist/mcp/tools.js +38 -24
- package/dist/packs/claude-code.js +22 -11
- package/dist/packs/cursor.js +9 -4
- package/dist/packs/gemini.js +9 -4
- package/dist/packs/json-config.js +41 -0
- package/dist/packs/omo.js +15 -6
- package/dist/request-status.js +81 -0
- package/dist/search/fts.js +93 -58
- package/dist/search/fusion.js +49 -47
- package/dist/search/guidance.js +18 -6
- package/dist/search/index.js +291 -70
- package/dist/search/lexical.js +71 -63
- package/dist/server.js +12 -0
- package/dist/skills/apply.js +130 -0
- package/dist/skills/capture.js +54 -0
- package/dist/skills/cloud-plan.js +57 -0
- package/dist/skills/cloud.js +81 -0
- package/dist/skills/fs.js +133 -0
- package/dist/skills/package.js +734 -0
- package/dist/skills/plan.js +83 -0
- package/dist/skills/receipt.js +44 -0
- package/dist/skills/rollback.js +119 -0
- package/dist/skills/transfer.js +51 -0
- package/dist/store/db.js +7 -1
- package/dist/store/execution.js +2 -34
- package/dist/store/export.js +25 -4
- package/dist/store/migrations/007_sync.sql +49 -0
- package/dist/store/migrations/008_connector_commands.sql +12 -0
- package/dist/store/queries.js +33 -7
- package/dist/store/writes.js +38 -5
- package/dist/sync/cloud-blobs.js +175 -0
- package/dist/sync/cloud-commands.js +101 -0
- package/dist/sync/cloud-connection.js +105 -0
- package/dist/sync/cloud-delivery.js +114 -0
- package/dist/sync/cloud-documents.js +103 -0
- package/dist/sync/cloud-export.js +67 -0
- package/dist/sync/cloud-http.js +150 -0
- package/dist/sync/cloud-jobs.js +94 -0
- package/dist/sync/cloud-pending.js +143 -0
- package/dist/sync/cloud-prepare.js +101 -0
- package/dist/sync/cloud-restore.js +136 -0
- package/dist/sync/cloud-rollback.js +24 -0
- package/dist/sync/cloud-runner.js +86 -0
- package/dist/sync/cloud-runtime.js +293 -0
- package/dist/sync/cloud-schema.js +32 -0
- package/dist/sync/cloud-stage.js +112 -0
- package/dist/sync/cloud-wire.js +693 -0
- package/dist/sync/cloud-worker.js +57 -0
- package/dist/sync/cloud.js +157 -0
- package/dist/sync/command-apply.js +140 -0
- package/dist/sync/command-control.js +25 -0
- package/dist/sync/command-wire.js +437 -0
- package/dist/sync/consent.js +130 -0
- package/dist/sync/journal.js +43 -0
- package/dist/sync/outbox.js +134 -0
- package/dist/sync/restore-rows.js +125 -0
- package/dist/sync/restore.js +174 -0
- package/dist/sync/snapshot.js +78 -0
- package/dist/sync/spool-request.js +103 -0
- package/dist/sync/tables.js +27 -0
- package/dist/sync/triggers.js +34 -0
- package/dist/workflow/compare.js +3 -43
- package/docs/cloud-sync.md +94 -0
- package/docs/private-skills.md +75 -0
- package/docs/sync-p4.md +102 -0
- package/docs/sync-restore.md +167 -0
- package/package.json +18 -5
- package/src/store/migrations/007_sync.sql +49 -0
- package/src/store/migrations/008_connector_commands.sql +12 -0
- package/dist/dashboard/assets/chunk-5DFWNJC2.js +0 -1
- package/dist/dashboard/assets/chunk-S6B2HJQA.js +0 -17
package/dist/store/writes.js
CHANGED
|
@@ -71,6 +71,14 @@ export function createWrites(ctx) {
|
|
|
71
71
|
const pendingInSessions = db.prepare(`SELECT DISTINCT e.* FROM json_each(?) s
|
|
72
72
|
CROSS JOIN load_candidates c ON c.session_id = s.value AND c.ts >= ? AND c.ts < ?
|
|
73
73
|
CROSS JOIN events e ON e.id = c.load_id WHERE ${pending}`);
|
|
74
|
+
const pendingBatch = db.prepare(`SELECT e.* FROM events e INDEXED BY events_pending_load_ts
|
|
75
|
+
WHERE ${pending} AND e.ts >= ? AND e.ts < ? AND (e.ts, e.rowid) > (?, ?)
|
|
76
|
+
ORDER BY e.ts, e.rowid LIMIT ?`);
|
|
77
|
+
const pendingSessionBatch = db.prepare(`SELECT DISTINCT e.rowid, e.* FROM json_each(?) s
|
|
78
|
+
CROSS JOIN load_candidates c ON c.session_id = s.value AND c.ts >= ? AND c.ts < ?
|
|
79
|
+
CROSS JOIN events e ON e.id = c.load_id
|
|
80
|
+
WHERE ${pending} AND (e.ts, e.rowid) > (?, ?) ORDER BY e.ts, e.rowid LIMIT ?`);
|
|
81
|
+
const repairCursor = db.prepare("SELECT rowid, ts, source, kind FROM events WHERE id = ?");
|
|
74
82
|
const eventById = db.prepare(`SELECT *, coalesce(json_extract(data_json, '$.skillarMcp'), json_extract(data_json, '$.gisoolMcp')) AS skillarMcp,
|
|
75
83
|
json_extract(data_json, '$.success') AS success FROM events WHERE id = ?`);
|
|
76
84
|
const loadById = db.prepare(`SELECT e.* FROM events e WHERE e.id = ? AND ${recoverable}`);
|
|
@@ -117,16 +125,32 @@ export function createWrites(ctx) {
|
|
|
117
125
|
.flatMap(({ value }) => loadsForTool.all(event.runtime, value, event.ts - URI_WINDOW_MS, event.ts + URI_WINDOW_MS, event.session_id));
|
|
118
126
|
}
|
|
119
127
|
}
|
|
128
|
+
else if (opts.limit !== undefined) {
|
|
129
|
+
const cursor = opts.after === undefined ? { ts: from, rowid: 0 }
|
|
130
|
+
: repairCursor.get(opts.after);
|
|
131
|
+
if (!cursor || cursor.ts < from || cursor.ts >= to
|
|
132
|
+
|| ("source" in cursor && (cursor.source !== "mcp" || cursor.kind !== "load"))) {
|
|
133
|
+
throw storeError("invalid_repair_cursor", "repair cursor must identify an MCP load in the requested time range");
|
|
134
|
+
}
|
|
135
|
+
loads = (opts.sessionIds === undefined
|
|
136
|
+
? pendingBatch.all(Math.max(from, cursor.ts), to, cursor.ts, cursor.rowid, opts.limit)
|
|
137
|
+
: pendingSessionBatch.all(JSON.stringify(opts.sessionIds), Math.max(from, cursor.ts), to, cursor.ts, cursor.rowid, opts.limit));
|
|
138
|
+
}
|
|
120
139
|
else {
|
|
121
140
|
loads = (opts.sessionIds === undefined ? pendingInRange.all(from, to)
|
|
122
141
|
: pendingInSessions.all(JSON.stringify(opts.sessionIds), from, to));
|
|
123
142
|
}
|
|
124
143
|
let count = 0;
|
|
144
|
+
let repaired = 0;
|
|
125
145
|
const uris = new Set();
|
|
126
146
|
if (!opts.readOnly && loads.length > 0)
|
|
127
147
|
db.exec("SAVEPOINT reconcile_loads");
|
|
128
148
|
try {
|
|
129
|
-
for (const
|
|
149
|
+
for (const selected of loads) {
|
|
150
|
+
// A repair CLI can run beside ingest. Read retained proof inside the transaction, not from the earlier batch.
|
|
151
|
+
const load = opts.limit === undefined || opts.readOnly ? selected : loadById.get(selected.id);
|
|
152
|
+
if (!load)
|
|
153
|
+
continue;
|
|
130
154
|
const uri = load.skill_uri;
|
|
131
155
|
if (uri === null)
|
|
132
156
|
continue;
|
|
@@ -159,8 +183,11 @@ export function createWrites(ctx) {
|
|
|
159
183
|
const json = JSON.stringify(data);
|
|
160
184
|
updateLoad.run(nextId, method, reason, json, load.id, nextId, method, reason, json);
|
|
161
185
|
}
|
|
162
|
-
if (sessionId !== undefined)
|
|
186
|
+
if (sessionId !== undefined) {
|
|
187
|
+
if (load.session_id === "unattributed")
|
|
188
|
+
repaired += 1;
|
|
163
189
|
continue;
|
|
190
|
+
}
|
|
164
191
|
count += 1;
|
|
165
192
|
uris.add(uri);
|
|
166
193
|
}
|
|
@@ -172,7 +199,9 @@ export function createWrites(ctx) {
|
|
|
172
199
|
db.exec("ROLLBACK TO reconcile_loads; RELEASE reconcile_loads");
|
|
173
200
|
throw error;
|
|
174
201
|
}
|
|
175
|
-
return { count, uris: [...uris].sort() }
|
|
202
|
+
return { count, uris: [...uris].sort(), ...(opts.limit === undefined ? {} : {
|
|
203
|
+
examined: loads.length, repaired, next: loads.length === opts.limit ? loads.at(-1).id : null,
|
|
204
|
+
}) };
|
|
176
205
|
},
|
|
177
206
|
append(event, attribution) {
|
|
178
207
|
const text = event.text ?? null;
|
|
@@ -212,9 +241,13 @@ export function createWrites(ctx) {
|
|
|
212
241
|
const existing = db.prepare(`SELECT * FROM sessions WHERE runtime = ? AND (runtime_session_id IN (${marks}) OR conversation_id IN (${marks}))
|
|
213
242
|
ORDER BY started_at LIMIT 1`).get(ref.runtime, ...keys, ...keys);
|
|
214
243
|
if (existing) {
|
|
244
|
+
const values = [startedAt, ref.conversationId ?? null, ref.runtimeSessionId ?? null, ref.clientName ?? null, ref.cwd ?? null];
|
|
215
245
|
db.prepare(`UPDATE sessions SET started_at = MIN(started_at, ?), conversation_id = COALESCE(conversation_id, ?),
|
|
216
|
-
runtime_session_id = COALESCE(runtime_session_id, ?), client_name = COALESCE(client_name, ?), cwd = COALESCE(cwd, ?)
|
|
217
|
-
|
|
246
|
+
runtime_session_id = COALESCE(runtime_session_id, ?), client_name = COALESCE(client_name, ?), cwd = COALESCE(cwd, ?)
|
|
247
|
+
WHERE id = ? AND (started_at IS NOT MIN(started_at, ?) OR conversation_id IS NOT COALESCE(conversation_id, ?)
|
|
248
|
+
OR runtime_session_id IS NOT COALESCE(runtime_session_id, ?) OR client_name IS NOT COALESCE(client_name, ?)
|
|
249
|
+
OR cwd IS NOT COALESCE(cwd, ?))`)
|
|
250
|
+
.run(...values, existing.id, ...values);
|
|
218
251
|
return existing.id;
|
|
219
252
|
}
|
|
220
253
|
}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import { randomUUID, createHash } from "node:crypto";
|
|
2
|
+
import { chmodSync, mkdirSync } from "node:fs";
|
|
3
|
+
import { dirname } from "node:path";
|
|
4
|
+
import { DatabaseSync } from "node:sqlite";
|
|
5
|
+
import { z } from "zod";
|
|
6
|
+
import { CLOUD_SCHEMA } from "./cloud-schema.js";
|
|
7
|
+
import { acknowledgeBatch, pendingBatch } from "./outbox.js";
|
|
8
|
+
import { initializeQueue, spoolRequest } from "./spool-request.js";
|
|
9
|
+
export class CloudSyncError extends Error {
|
|
10
|
+
code;
|
|
11
|
+
constructor(code) {
|
|
12
|
+
super(code);
|
|
13
|
+
this.code = code;
|
|
14
|
+
this.name = "CloudSyncError";
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
const digest = z.string().regex(/^[a-f0-9]{64}$/);
|
|
18
|
+
const leaseSchema = z.object({ owner: z.string(), epoch: z.number().int(), until_ms: z.number().int() });
|
|
19
|
+
/** Separate durable blob DB keeps large serialization/fsync out of source commits. */
|
|
20
|
+
export class CloudBlobs {
|
|
21
|
+
maxBytes;
|
|
22
|
+
#db;
|
|
23
|
+
#owner = randomUUID();
|
|
24
|
+
#epoch = null;
|
|
25
|
+
constructor(path, maxBytes = 64 * 1024 * 1024) {
|
|
26
|
+
this.maxBytes = maxBytes;
|
|
27
|
+
z.number().int().positive().parse(maxBytes);
|
|
28
|
+
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
29
|
+
this.#db = new DatabaseSync(path);
|
|
30
|
+
chmodSync(path, 0o600);
|
|
31
|
+
this.#db.exec(`PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL;
|
|
32
|
+
PRAGMA busy_timeout=5000; PRAGMA foreign_keys=ON`);
|
|
33
|
+
const version = z.object({ user_version: z.number() })
|
|
34
|
+
.parse(this.#db.prepare("PRAGMA user_version").get()).user_version;
|
|
35
|
+
if (version === 0) {
|
|
36
|
+
this.#db.exec(`BEGIN IMMEDIATE;
|
|
37
|
+
${CLOUD_SCHEMA}
|
|
38
|
+
PRAGMA user_version=1;
|
|
39
|
+
COMMIT`);
|
|
40
|
+
}
|
|
41
|
+
else if (version !== 1) {
|
|
42
|
+
this.#db.close();
|
|
43
|
+
throw new CloudSyncError("unsupported_spool_version");
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
acquire(now = Date.now(), ttlMs = 60_000) {
|
|
47
|
+
z.number().int().positive().parse(ttlMs);
|
|
48
|
+
const row = this.#db.prepare(`UPDATE lease SET
|
|
49
|
+
epoch=CASE WHEN owner=? THEN epoch ELSE epoch+1 END,owner=?,until_ms=?
|
|
50
|
+
WHERE id=1 AND (owner=? OR until_ms<=?) RETURNING owner,epoch,until_ms`)
|
|
51
|
+
.get(this.#owner, this.#owner, now + ttlMs, this.#owner, now);
|
|
52
|
+
if (row === undefined)
|
|
53
|
+
return false;
|
|
54
|
+
this.#epoch = leaseSchema.parse(row).epoch;
|
|
55
|
+
return true;
|
|
56
|
+
}
|
|
57
|
+
assertLease(now = Date.now()) {
|
|
58
|
+
const row = leaseSchema.parse(this.#db.prepare("SELECT * FROM lease WHERE id=1").get());
|
|
59
|
+
if (row.owner !== this.#owner || row.epoch !== this.#epoch || row.until_ms <= now) {
|
|
60
|
+
throw new CloudSyncError("spool_lease_lost");
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
renew(now = Date.now(), ttlMs = 60_000) {
|
|
64
|
+
const changed = this.#db.prepare(`UPDATE lease SET until_ms=?
|
|
65
|
+
WHERE id=1 AND owner=? AND epoch=? AND until_ms>?`)
|
|
66
|
+
.run(now + ttlMs, this.#owner, this.#epoch, now);
|
|
67
|
+
if (changed.changes !== 1)
|
|
68
|
+
throw new CloudSyncError("spool_lease_lost");
|
|
69
|
+
}
|
|
70
|
+
release() {
|
|
71
|
+
if (this.#epoch !== null) {
|
|
72
|
+
this.#db.prepare("UPDATE lease SET until_ms=0 WHERE id=1 AND owner=? AND epoch=?")
|
|
73
|
+
.run(this.#owner, this.#epoch);
|
|
74
|
+
this.#epoch = null;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
#transaction(run) {
|
|
78
|
+
this.#db.exec("BEGIN IMMEDIATE");
|
|
79
|
+
try {
|
|
80
|
+
this.assertLease();
|
|
81
|
+
const result = run();
|
|
82
|
+
this.#db.exec("COMMIT");
|
|
83
|
+
return result;
|
|
84
|
+
}
|
|
85
|
+
catch (error) {
|
|
86
|
+
this.#db.exec("ROLLBACK");
|
|
87
|
+
throw error;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
put(bytes) {
|
|
91
|
+
return this.#transaction(() => {
|
|
92
|
+
const hash = createHash("sha256").update(bytes).digest("hex");
|
|
93
|
+
if (this.#db.prepare("SELECT 1 FROM blobs WHERE hash=?").get(hash) !== undefined) {
|
|
94
|
+
return { hash, bytes: bytes.byteLength };
|
|
95
|
+
}
|
|
96
|
+
const used = z.object({ bytes: z.number() }).parse(this.#db.prepare(`SELECT COALESCE((SELECT sum(nbytes) FROM blobs),0)+
|
|
97
|
+
COALESCE((SELECT sum(payload_bytes) FROM sync_outbox),0) AS bytes`).get());
|
|
98
|
+
if (used.bytes + bytes.byteLength > this.maxBytes)
|
|
99
|
+
throw new CloudSyncError("outbox_pressure");
|
|
100
|
+
this.#db.prepare("INSERT INTO blobs VALUES(?,?,?)").run(hash, bytes, bytes.byteLength);
|
|
101
|
+
return { hash, bytes: bytes.byteLength };
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
get(hash) {
|
|
105
|
+
digest.parse(hash);
|
|
106
|
+
const row = this.#db.prepare("SELECT bytes,nbytes FROM blobs WHERE hash=?").get(hash);
|
|
107
|
+
if (row === undefined)
|
|
108
|
+
throw new CloudSyncError("spool_blob_missing");
|
|
109
|
+
const saved = z.object({
|
|
110
|
+
bytes: z.instanceof(Uint8Array), nbytes: z.number().int().nonnegative(),
|
|
111
|
+
}).parse(row);
|
|
112
|
+
if (saved.bytes.byteLength !== saved.nbytes ||
|
|
113
|
+
createHash("sha256").update(saved.bytes).digest("hex") !== hash) {
|
|
114
|
+
throw new CloudSyncError("spool_blob_corrupt");
|
|
115
|
+
}
|
|
116
|
+
return saved.bytes;
|
|
117
|
+
}
|
|
118
|
+
prune(retained = new Set()) {
|
|
119
|
+
return this.#transaction(() => {
|
|
120
|
+
const rows = z.array(z.object({ hash: digest }))
|
|
121
|
+
.parse(this.#db.prepare(`SELECT hash FROM blobs b WHERE NOT EXISTS
|
|
122
|
+
(SELECT 1 FROM sync_outbox_blobs r WHERE r.hash=b.hash)`).all());
|
|
123
|
+
const remove = this.#db.prepare("DELETE FROM blobs WHERE hash=?");
|
|
124
|
+
let deleted = 0;
|
|
125
|
+
for (const row of rows)
|
|
126
|
+
if (!retained.has(row.hash)) {
|
|
127
|
+
remove.run(row.hash);
|
|
128
|
+
deleted++;
|
|
129
|
+
}
|
|
130
|
+
return deleted;
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
initialize(input) {
|
|
134
|
+
this.#transaction(() => initializeQueue(this.#db, input));
|
|
135
|
+
}
|
|
136
|
+
enqueue(input) {
|
|
137
|
+
return spoolRequest(this.#db, {
|
|
138
|
+
...input,
|
|
139
|
+
quota: {
|
|
140
|
+
maxBatches: input.quota?.maxBatches ?? 1024,
|
|
141
|
+
maxBytes: Math.min(input.quota?.maxBytes ?? this.maxBytes, this.maxBytes),
|
|
142
|
+
},
|
|
143
|
+
assertLease: () => this.assertLease(),
|
|
144
|
+
storedBlobBytes: () => z.object({ bytes: z.number().int().nonnegative() })
|
|
145
|
+
.parse(this.#db.prepare("SELECT COALESCE(sum(nbytes),0) AS bytes FROM blobs").get()).bytes,
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
pending(streamId) {
|
|
149
|
+
return pendingBatch(this.#db, streamId);
|
|
150
|
+
}
|
|
151
|
+
pendingPrefix(prefix) {
|
|
152
|
+
const row = this.#db.prepare(`SELECT stream_id FROM sync_outbox
|
|
153
|
+
WHERE substr(stream_id,1,length(?))=? ORDER BY rowid LIMIT 1`).get(prefix, prefix);
|
|
154
|
+
if (row === undefined)
|
|
155
|
+
return null;
|
|
156
|
+
const { stream_id: streamId } = z.object({ stream_id: z.string() }).parse(row);
|
|
157
|
+
const batch = pendingBatch(this.#db, streamId);
|
|
158
|
+
if (batch === null)
|
|
159
|
+
throw new CloudSyncError("spool_sequence_gap");
|
|
160
|
+
return batch;
|
|
161
|
+
}
|
|
162
|
+
acknowledge(receipt) {
|
|
163
|
+
acknowledgeBatch(this.#db, receipt, () => this.assertLease());
|
|
164
|
+
}
|
|
165
|
+
cursor(streamId) {
|
|
166
|
+
const row = this.#db.prepare("SELECT ack_sequence,ack_cut FROM sync_streams WHERE stream_id=?").get(streamId);
|
|
167
|
+
return row === undefined ? null : z.object({
|
|
168
|
+
ack_sequence: z.number().int().nonnegative(), ack_cut: z.number().int().nonnegative(),
|
|
169
|
+
}).parse(row);
|
|
170
|
+
}
|
|
171
|
+
close() {
|
|
172
|
+
this.release();
|
|
173
|
+
this.#db.close();
|
|
174
|
+
}
|
|
175
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
import { contentKinds } from "./cloud-wire.js";
|
|
4
|
+
import { CloudSyncError } from "./cloud-blobs.js";
|
|
5
|
+
import { atomicCloudJson, currentCloudDevice, readConnection } from "./cloud-connection.js";
|
|
6
|
+
import { CloudHttp, CloudRequestError } from "./cloud-http.js";
|
|
7
|
+
import { sourceCommandsEnabled } from "./command-control.js";
|
|
8
|
+
import { commandPollResponseSchema, commandResultSchema, commandStatusSchema, sourceCommandSchema } from "./command-wire.js";
|
|
9
|
+
export const commandApplyRequestSchema = z.strictObject({
|
|
10
|
+
kind: z.literal("apply-command"), id: z.string().uuid(), instanceId: z.string().uuid(),
|
|
11
|
+
command: sourceCommandSchema,
|
|
12
|
+
});
|
|
13
|
+
const commandApplyReplySchema = z.strictObject({
|
|
14
|
+
kind: z.literal("command-applied"), id: z.string().uuid(), result: commandResultSchema.nullable(),
|
|
15
|
+
});
|
|
16
|
+
export function requestCommandApply(port, command, instanceId) {
|
|
17
|
+
if (port === null)
|
|
18
|
+
return Promise.resolve(null);
|
|
19
|
+
const id = randomUUID();
|
|
20
|
+
return new Promise(resolve => {
|
|
21
|
+
const finish = (result) => {
|
|
22
|
+
clearTimeout(timeout);
|
|
23
|
+
port.off("message", reply);
|
|
24
|
+
resolve(result);
|
|
25
|
+
};
|
|
26
|
+
const reply = (value) => {
|
|
27
|
+
const parsed = commandApplyReplySchema.safeParse(value);
|
|
28
|
+
if (parsed.success && parsed.data.id === id)
|
|
29
|
+
finish(parsed.data.result);
|
|
30
|
+
};
|
|
31
|
+
const timeout = setTimeout(() => finish(null), 5000);
|
|
32
|
+
port.on("message", reply);
|
|
33
|
+
port.postMessage({ kind: "apply-command", id, instanceId, command });
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
/** One bounded poll/apply/ACK in the existing sync turn; no new persistent transport. */
|
|
37
|
+
export async function runCloudCommandStep(home, progress, apply, options = {}) {
|
|
38
|
+
if (progress.phase !== "caught-up" || progress.sourceDomain === undefined || !sourceCommandsEnabled(home))
|
|
39
|
+
return progress;
|
|
40
|
+
const connection = readConnection(home);
|
|
41
|
+
if (connection === null || connection.instanceId !== progress.instanceId)
|
|
42
|
+
return progress;
|
|
43
|
+
const save = (state, code = null, commandId = null) => atomicCloudJson(home, "command-status.json", { state, code, commandId, updatedAtMs: Date.now() });
|
|
44
|
+
try {
|
|
45
|
+
const self = await currentCloudDevice(connection, options.fetcher);
|
|
46
|
+
if (!self.ingestEnabled || !self.scopes.includes("activity:read") || !self.scopes.includes("activity:ingest") ||
|
|
47
|
+
self.consent.mode !== "managed-content" || self.consent.scope.rootIds.length !== 0 ||
|
|
48
|
+
!self.consent.scope.projectIds.includes(`device:${connection.deviceId}`) ||
|
|
49
|
+
!contentKinds.every(kind => self.consent.mode === "managed-content" && self.consent.contentKinds.includes(kind))) {
|
|
50
|
+
save("disabled", "command_consent_required");
|
|
51
|
+
return progress;
|
|
52
|
+
}
|
|
53
|
+
const http = new CloudHttp(connection.origin, connection.token, options.fetcher);
|
|
54
|
+
const { command } = commandPollResponseSchema.parse(await http.request("/v1/activity/commands/poll", {
|
|
55
|
+
body: JSON.stringify({ sourceDomain: progress.sourceDomain }),
|
|
56
|
+
}));
|
|
57
|
+
if (command === null) {
|
|
58
|
+
save("idle");
|
|
59
|
+
return progress;
|
|
60
|
+
}
|
|
61
|
+
if (command.accountId !== connection.accountId || command.generation !== connection.generation ||
|
|
62
|
+
command.deviceId !== connection.deviceId || command.sourceDomain !== progress.sourceDomain ||
|
|
63
|
+
command.consentVersion !== self.consent.version)
|
|
64
|
+
throw new CloudSyncError("command_identity_mismatch");
|
|
65
|
+
// A disable, disconnect or consent change during poll must win before local apply.
|
|
66
|
+
const current = await currentCloudDevice(connection, options.fetcher);
|
|
67
|
+
if (!sourceCommandsEnabled(home) || readConnection(home)?.instanceId !== connection.instanceId ||
|
|
68
|
+
JSON.stringify(current) !== JSON.stringify(self)) {
|
|
69
|
+
save("disabled", "command_authority_changed", command.commandId);
|
|
70
|
+
return progress;
|
|
71
|
+
}
|
|
72
|
+
const result = await apply(command, connection.instanceId);
|
|
73
|
+
if (result === null) {
|
|
74
|
+
save("pending", "source_apply_unavailable", command.commandId);
|
|
75
|
+
return progress;
|
|
76
|
+
}
|
|
77
|
+
const receipt = commandResultSchema.parse(result);
|
|
78
|
+
if (receipt.commandId !== command.commandId || receipt.payloadHash !== command.payloadHash) {
|
|
79
|
+
throw new CloudSyncError("command_receipt_mismatch");
|
|
80
|
+
}
|
|
81
|
+
const acknowledged = commandStatusSchema.parse(await http.request("/v1/activity/commands/ack", {
|
|
82
|
+
body: JSON.stringify({ sourceDomain: command.sourceDomain, ...receipt }),
|
|
83
|
+
}));
|
|
84
|
+
if (acknowledged.commandId !== receipt.commandId || acknowledged.state !== receipt.state ||
|
|
85
|
+
acknowledged.sourceRevision !== receipt.sourceRevision)
|
|
86
|
+
throw new CloudSyncError("command_ack_mismatch");
|
|
87
|
+
save(receipt.state, receipt.error, command.commandId);
|
|
88
|
+
return progress;
|
|
89
|
+
}
|
|
90
|
+
catch (failure) {
|
|
91
|
+
if (failure instanceof CloudRequestError) {
|
|
92
|
+
save(failure.state, failure.code);
|
|
93
|
+
return progress;
|
|
94
|
+
}
|
|
95
|
+
if (failure instanceof CloudSyncError) {
|
|
96
|
+
save("quarantined", failure.code);
|
|
97
|
+
return progress;
|
|
98
|
+
}
|
|
99
|
+
throw failure;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeFileSync, } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
import { CloudSyncError } from "./cloud-blobs.js";
|
|
6
|
+
import { CloudHttp, cloudOriginSchema } from "./cloud-http.js";
|
|
7
|
+
import { consentSchema } from "./cloud-wire.js";
|
|
8
|
+
export const deviceSelfSchema = z.strictObject({
|
|
9
|
+
accountId: z.string().min(1), deviceId: z.string().min(1),
|
|
10
|
+
generation: z.number().int().positive(), shardId: z.literal("activity-00"),
|
|
11
|
+
scopes: z.array(z.enum(["activity:read", "activity:ingest", "skills:read", "skills:write"])).min(1),
|
|
12
|
+
ingestEnabled: z.boolean(), version: z.number().int().nonnegative(),
|
|
13
|
+
consent: consentSchema,
|
|
14
|
+
});
|
|
15
|
+
export const connectionSchema = z.strictObject({
|
|
16
|
+
version: z.literal(1), instanceId: z.string().uuid(), origin: cloudOriginSchema,
|
|
17
|
+
accountId: z.string().min(1), deviceId: z.string().min(1),
|
|
18
|
+
generation: z.number().int().positive(), token: z.string().regex(/^[A-Za-z0-9_-]{43}$/),
|
|
19
|
+
expiresAt: z.number().int().positive(),
|
|
20
|
+
// Last scopes the server confirmed for this device; absent in files written
|
|
21
|
+
// before scopes were persisted, where the caller probes or assumes instead.
|
|
22
|
+
scopes: z.array(z.enum(["activity:read", "activity:ingest", "skills:read", "skills:write"]))
|
|
23
|
+
.min(1).optional(),
|
|
24
|
+
});
|
|
25
|
+
export function cloudHome(home) {
|
|
26
|
+
return join(home, "cloud");
|
|
27
|
+
}
|
|
28
|
+
export function atomicCloudJson(home, name, value) {
|
|
29
|
+
if (!/^[a-z0-9-]+\.json$/.test(name))
|
|
30
|
+
throw new CloudSyncError("invalid_state_path");
|
|
31
|
+
const directory = cloudHome(home);
|
|
32
|
+
mkdirSync(directory, { recursive: true, mode: 0o700 });
|
|
33
|
+
const target = join(directory, name), temporary = `${target}.${randomUUID()}.tmp`;
|
|
34
|
+
const fd = openSync(temporary, "wx", 0o600);
|
|
35
|
+
try {
|
|
36
|
+
writeFileSync(fd, JSON.stringify(value));
|
|
37
|
+
fsyncSync(fd);
|
|
38
|
+
}
|
|
39
|
+
finally {
|
|
40
|
+
closeSync(fd);
|
|
41
|
+
}
|
|
42
|
+
renameSync(temporary, target);
|
|
43
|
+
const parent = openSync(directory, "r");
|
|
44
|
+
try {
|
|
45
|
+
fsyncSync(parent);
|
|
46
|
+
}
|
|
47
|
+
finally {
|
|
48
|
+
closeSync(parent);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
export function readConnection(home) {
|
|
52
|
+
const path = join(cloudHome(home), "connection.json");
|
|
53
|
+
if (!existsSync(path))
|
|
54
|
+
return null;
|
|
55
|
+
const input = JSON.parse(readFileSync(path, "utf8"));
|
|
56
|
+
return connectionSchema.parse(input);
|
|
57
|
+
}
|
|
58
|
+
export function disconnectCloud(home) {
|
|
59
|
+
const path = join(cloudHome(home), "connection.json");
|
|
60
|
+
if (existsSync(path))
|
|
61
|
+
unlinkSync(path);
|
|
62
|
+
// Source journal, immutable pending requests, blobs and local backups survive.
|
|
63
|
+
}
|
|
64
|
+
/** Connection exchanges a browser-issued grant; it never grants upload consent. */
|
|
65
|
+
export async function connectCloud(home, input, fetcher = fetch) {
|
|
66
|
+
const existing = readConnection(home);
|
|
67
|
+
if (existing !== null &&
|
|
68
|
+
(input.replaceDeviceId === undefined || existing.deviceId !== input.replaceDeviceId)) {
|
|
69
|
+
throw new CloudSyncError("already_connected");
|
|
70
|
+
}
|
|
71
|
+
const exchange = new CloudHttp(input.origin, null, fetcher);
|
|
72
|
+
const credential = z.strictObject({
|
|
73
|
+
token: z.string().regex(/^[A-Za-z0-9_-]{43}$/), deviceId: z.string().min(1),
|
|
74
|
+
expiresAt: z.number().int().positive(),
|
|
75
|
+
}).parse(await exchange.request("/v1/devices/exchange", {
|
|
76
|
+
body: JSON.stringify({ grant: z.string().min(32).max(128).parse(input.grant) }),
|
|
77
|
+
}));
|
|
78
|
+
const self = deviceSelfSchema.parse(await new CloudHttp(input.origin, credential.token, fetcher)
|
|
79
|
+
.request("/v1/devices/self"));
|
|
80
|
+
if (self.deviceId !== credential.deviceId)
|
|
81
|
+
throw new CloudSyncError("device_identity_mismatch");
|
|
82
|
+
// A scope upgrade must land on the device it was approved for, nothing else.
|
|
83
|
+
if (input.replaceDeviceId !== undefined && credential.deviceId !== input.replaceDeviceId) {
|
|
84
|
+
throw new CloudSyncError("device_identity_mismatch");
|
|
85
|
+
}
|
|
86
|
+
const connection = connectionSchema.parse({
|
|
87
|
+
version: 1, instanceId: randomUUID(), origin: input.origin, accountId: self.accountId,
|
|
88
|
+
deviceId: self.deviceId, generation: self.generation,
|
|
89
|
+
token: credential.token, expiresAt: credential.expiresAt, scopes: [...self.scopes],
|
|
90
|
+
});
|
|
91
|
+
atomicCloudJson(home, "connection.json", connection);
|
|
92
|
+
const { token: _secret, ...visible } = connection;
|
|
93
|
+
return visible;
|
|
94
|
+
}
|
|
95
|
+
export async function currentCloudDevice(connection, fetcher = fetch, attempt = 0) {
|
|
96
|
+
if (connection.expiresAt <= Math.floor(Date.now() / 1000)) {
|
|
97
|
+
throw new CloudSyncError("credentials_expired");
|
|
98
|
+
}
|
|
99
|
+
const self = deviceSelfSchema.parse(await new CloudHttp(connection.origin, connection.token, fetcher, 30_000, attempt)
|
|
100
|
+
.request("/v1/devices/self"));
|
|
101
|
+
if (self.deviceId !== connection.deviceId || self.accountId !== connection.accountId ||
|
|
102
|
+
self.generation !== connection.generation)
|
|
103
|
+
throw new CloudSyncError("device_identity_mismatch");
|
|
104
|
+
return self;
|
|
105
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { CloudSyncError } from "./cloud-blobs.js";
|
|
3
|
+
import { sha256 } from "./cloud-documents.js";
|
|
4
|
+
import { CloudRequestError } from "./cloud-http.js";
|
|
5
|
+
import { requireJob } from "./cloud-jobs.js";
|
|
6
|
+
import { contentChunkRefSchema, migrationPageSchema, migrationStatusSchema, parseContentBatch, parseContentChunkUpload, } from "./cloud-wire.js";
|
|
7
|
+
export const receiptSchema = z.strictObject({
|
|
8
|
+
batchId: z.string(), payloadHash: z.string().regex(/^[a-f0-9]{64}$/),
|
|
9
|
+
sequence: z.number().int().positive(), durableAt: z.number().int(),
|
|
10
|
+
projectedThrough: z.number().int().nonnegative(), changeSeq: z.number().int().positive(),
|
|
11
|
+
state: z.literal("projected"),
|
|
12
|
+
});
|
|
13
|
+
async function request(blobs, http, path, body) {
|
|
14
|
+
blobs.renew();
|
|
15
|
+
const value = await http.request(path, { body });
|
|
16
|
+
blobs.renew();
|
|
17
|
+
return value;
|
|
18
|
+
}
|
|
19
|
+
async function upload(blobs, http, refs, expected) {
|
|
20
|
+
if (refs.length !== expected.length)
|
|
21
|
+
throw new CloudSyncError("spool_blob_corrupt");
|
|
22
|
+
for (const [index, reference] of refs.entries()) {
|
|
23
|
+
const bytes = blobs.get(reference.hash);
|
|
24
|
+
if (bytes.byteLength !== reference.bytes)
|
|
25
|
+
throw new CloudSyncError("spool_blob_corrupt");
|
|
26
|
+
const body = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
27
|
+
const chunk = parseContentChunkUpload(JSON.parse(body));
|
|
28
|
+
const declared = contentChunkRefSchema.parse({
|
|
29
|
+
index: chunk.index, revision: chunk.revision,
|
|
30
|
+
plaintextHash: chunk.plaintextHash, bytes: chunk.bytes,
|
|
31
|
+
});
|
|
32
|
+
// The strict reference parser rejects extra chunk fields; select only the
|
|
33
|
+
// machine-consumed identity before comparing the server's durable chunk ACK.
|
|
34
|
+
const wanted = expected[index];
|
|
35
|
+
const ack = z.strictObject({ stored: z.literal(true), chunk: contentChunkRefSchema })
|
|
36
|
+
.parse(await request(blobs, http, "/v1/activity/chunks", body));
|
|
37
|
+
if (wanted === undefined || JSON.stringify(ack.chunk) !== JSON.stringify(wanted) ||
|
|
38
|
+
declared.index !== wanted.index || declared.plaintextHash !== wanted.plaintextHash ||
|
|
39
|
+
declared.bytes !== wanted.bytes || declared.revision !== wanted.revision) {
|
|
40
|
+
throw new CloudSyncError("invalid_remote_receipt");
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
/** A bounded unit of work; an accepted snapshot is never inferred from a page ACK. */
|
|
45
|
+
export async function deliverCloudJob(blobs, http, batch) {
|
|
46
|
+
const job = requireJob(batch.payload);
|
|
47
|
+
switch (job.kind) {
|
|
48
|
+
case "cdc": {
|
|
49
|
+
const body = parseContentBatch(JSON.parse(job.body));
|
|
50
|
+
if (body.sequence !== batch.sequence || body.batchId !== batch.batch_id ||
|
|
51
|
+
body.journal.from !== batch.journal_from || body.journal.through !== batch.journal_through ||
|
|
52
|
+
sha256(JSON.stringify(body)) !== job.payloadHash)
|
|
53
|
+
throw new CloudSyncError("invalid_spool_job");
|
|
54
|
+
try {
|
|
55
|
+
await upload(blobs, http, job.refs, body.chunks);
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
// Expired chunks can belong to an accepted batch whose ACK was lost.
|
|
59
|
+
// Only the exact receipt below can acknowledge it; otherwise stay pending.
|
|
60
|
+
if (!(error instanceof CloudRequestError && error.status === 409 && error.code === "content_expired")) {
|
|
61
|
+
throw error;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
const receipt = receiptSchema.parse(await request(blobs, http, "/v1/activity/batches", job.body));
|
|
65
|
+
if (receipt.batchId !== body.batchId || receipt.sequence !== body.sequence ||
|
|
66
|
+
receipt.payloadHash !== job.payloadHash || receipt.projectedThrough !== body.journal.through) {
|
|
67
|
+
throw new CloudSyncError("invalid_remote_receipt");
|
|
68
|
+
}
|
|
69
|
+
return { accepted: true, kind: "cdc", excluded: job.excluded, receipt };
|
|
70
|
+
}
|
|
71
|
+
case "migration": {
|
|
72
|
+
const status = migrationStatusSchema.parse(await request(blobs, http, "/v1/activity/migrations/begin", JSON.stringify(job.begin)));
|
|
73
|
+
if (JSON.stringify(status.migration) !== JSON.stringify(job.begin)) {
|
|
74
|
+
throw new CloudSyncError("invalid_remote_receipt");
|
|
75
|
+
}
|
|
76
|
+
if (status.state === "abandoned")
|
|
77
|
+
throw new CloudSyncError("migration_closed");
|
|
78
|
+
const reference = {
|
|
79
|
+
deviceId: job.begin.deviceId, generation: job.begin.generation,
|
|
80
|
+
sourceDomain: job.begin.sourceDomain, snapshotId: job.begin.snapshotId,
|
|
81
|
+
};
|
|
82
|
+
const staged = new Map(status.stagedPages.map(page => [page.index, page]));
|
|
83
|
+
for (const page of job.pages) {
|
|
84
|
+
const previous = staged.get(page.page.index);
|
|
85
|
+
if (previous !== undefined && JSON.stringify(previous) !== JSON.stringify(page.page)) {
|
|
86
|
+
throw new CloudSyncError("invalid_remote_receipt");
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
const next = job.pages.find(page => !staged.has(page.page.index));
|
|
90
|
+
if (next !== undefined) {
|
|
91
|
+
await upload(blobs, http, next.refs, next.page.chunks);
|
|
92
|
+
const saved = z.strictObject({ staged: z.literal(true), page: migrationPageSchema })
|
|
93
|
+
.parse(await request(blobs, http, "/v1/activity/migrations/page", JSON.stringify(next.page)));
|
|
94
|
+
if (JSON.stringify(saved.page) !== JSON.stringify(next.page)) {
|
|
95
|
+
throw new CloudSyncError("invalid_remote_receipt");
|
|
96
|
+
}
|
|
97
|
+
return { accepted: false, kind: "migration", excluded: job.excluded,
|
|
98
|
+
stagedPages: staged.size + 1, totalPages: job.pages.length };
|
|
99
|
+
}
|
|
100
|
+
const completed = status.accepted ? status : migrationStatusSchema.parse(await request(blobs, http, "/v1/activity/migrations/complete", JSON.stringify(reference)));
|
|
101
|
+
if (!completed.accepted || JSON.stringify(completed.migration) !== JSON.stringify(job.begin) ||
|
|
102
|
+
(job.begin.purpose === "initial" &&
|
|
103
|
+
(completed.cursor.sequence !== 0 || completed.cursor.journalThrough !== job.begin.cut))) {
|
|
104
|
+
throw new CloudSyncError("invalid_remote_receipt");
|
|
105
|
+
}
|
|
106
|
+
return { accepted: true, kind: "migration", excluded: job.excluded,
|
|
107
|
+
status: completed };
|
|
108
|
+
}
|
|
109
|
+
default: {
|
|
110
|
+
const exhaustive = job;
|
|
111
|
+
return exhaustive;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|