skillar 0.2.1 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/NOTICE +280 -0
  2. package/README.md +37 -5
  3. package/dist/analysis/finding.js +8 -190
  4. package/dist/analysis/kernel.js +896 -0
  5. package/dist/analysis/tree.js +7 -154
  6. package/dist/analysis/usage.js +9 -394
  7. package/dist/api/flows.js +42 -212
  8. package/dist/api/report-kernel.js +1063 -0
  9. package/dist/api/routes.js +12 -2
  10. package/dist/catalog/fields.js +138 -146
  11. package/dist/catalog/watcher.js +7 -2
  12. package/dist/cli/init.js +45 -0
  13. package/dist/cli/main.js +76 -3
  14. package/dist/cli/private-skills.js +122 -0
  15. package/dist/cli/serve.js +15 -1
  16. package/dist/cli/sync.js +124 -0
  17. package/dist/dashboard/NOTICE +280 -0
  18. package/dist/dashboard/assets/app.js +737 -275
  19. package/dist/dashboard/assets/astryx-reset.css +407 -0
  20. package/dist/dashboard/assets/astryx.css +1077 -0
  21. package/dist/dashboard/assets/charts.css +54 -0
  22. package/dist/dashboard/assets/chunk-TPGDA2J4.js +1 -0
  23. package/dist/dashboard/assets/chunk-YLWHVDMP.js +36 -0
  24. package/dist/dashboard/assets/controls.css +207 -0
  25. package/dist/dashboard/assets/overlays.css +132 -0
  26. package/dist/dashboard/assets/p1.css +135 -0
  27. package/dist/dashboard/assets/shell.css +71 -0
  28. package/dist/dashboard/assets/skillar-theme.css +692 -0
  29. package/dist/dashboard/assets/styles.css +224 -121
  30. package/dist/dashboard/assets/tokens.css +278 -0
  31. package/dist/dashboard/assets/widgets.css +83 -0
  32. package/dist/dashboard/index.html +11 -2
  33. package/dist/dashboard/viewers/viewers.css +61 -0
  34. package/dist/ingest/attribution.js +5 -1
  35. package/dist/ingest/pipeline.js +2 -0
  36. package/dist/ingest/request-recorder.js +39 -0
  37. package/dist/mcp/dispatch.js +2 -2
  38. package/dist/mcp/http.js +43 -3
  39. package/dist/mcp/skills-extension.js +42 -15
  40. package/dist/mcp/stdio-proxy.js +24 -3
  41. package/dist/mcp/tools.js +38 -24
  42. package/dist/packs/claude-code.js +22 -11
  43. package/dist/packs/cursor.js +9 -4
  44. package/dist/packs/gemini.js +9 -4
  45. package/dist/packs/json-config.js +41 -0
  46. package/dist/packs/omo.js +15 -6
  47. package/dist/request-status.js +81 -0
  48. package/dist/search/fts.js +93 -58
  49. package/dist/search/fusion.js +49 -47
  50. package/dist/search/guidance.js +18 -6
  51. package/dist/search/index.js +291 -70
  52. package/dist/search/lexical.js +71 -63
  53. package/dist/server.js +12 -0
  54. package/dist/skills/apply.js +130 -0
  55. package/dist/skills/capture.js +54 -0
  56. package/dist/skills/cloud-plan.js +57 -0
  57. package/dist/skills/cloud.js +81 -0
  58. package/dist/skills/fs.js +133 -0
  59. package/dist/skills/package.js +734 -0
  60. package/dist/skills/plan.js +83 -0
  61. package/dist/skills/receipt.js +44 -0
  62. package/dist/skills/rollback.js +119 -0
  63. package/dist/skills/transfer.js +51 -0
  64. package/dist/store/db.js +7 -1
  65. package/dist/store/execution.js +2 -34
  66. package/dist/store/export.js +25 -4
  67. package/dist/store/migrations/007_sync.sql +49 -0
  68. package/dist/store/migrations/008_connector_commands.sql +12 -0
  69. package/dist/store/queries.js +33 -7
  70. package/dist/store/writes.js +38 -5
  71. package/dist/sync/cloud-blobs.js +175 -0
  72. package/dist/sync/cloud-commands.js +101 -0
  73. package/dist/sync/cloud-connection.js +105 -0
  74. package/dist/sync/cloud-delivery.js +114 -0
  75. package/dist/sync/cloud-documents.js +103 -0
  76. package/dist/sync/cloud-export.js +67 -0
  77. package/dist/sync/cloud-http.js +150 -0
  78. package/dist/sync/cloud-jobs.js +94 -0
  79. package/dist/sync/cloud-pending.js +143 -0
  80. package/dist/sync/cloud-prepare.js +101 -0
  81. package/dist/sync/cloud-restore.js +136 -0
  82. package/dist/sync/cloud-rollback.js +24 -0
  83. package/dist/sync/cloud-runner.js +86 -0
  84. package/dist/sync/cloud-runtime.js +293 -0
  85. package/dist/sync/cloud-schema.js +32 -0
  86. package/dist/sync/cloud-stage.js +112 -0
  87. package/dist/sync/cloud-wire.js +693 -0
  88. package/dist/sync/cloud-worker.js +57 -0
  89. package/dist/sync/cloud.js +157 -0
  90. package/dist/sync/command-apply.js +140 -0
  91. package/dist/sync/command-control.js +25 -0
  92. package/dist/sync/command-wire.js +437 -0
  93. package/dist/sync/consent.js +130 -0
  94. package/dist/sync/journal.js +43 -0
  95. package/dist/sync/outbox.js +134 -0
  96. package/dist/sync/restore-rows.js +125 -0
  97. package/dist/sync/restore.js +174 -0
  98. package/dist/sync/snapshot.js +78 -0
  99. package/dist/sync/spool-request.js +103 -0
  100. package/dist/sync/tables.js +27 -0
  101. package/dist/sync/triggers.js +34 -0
  102. package/dist/workflow/compare.js +3 -43
  103. package/docs/cloud-sync.md +94 -0
  104. package/docs/private-skills.md +75 -0
  105. package/docs/sync-p4.md +102 -0
  106. package/docs/sync-restore.md +167 -0
  107. package/package.json +18 -5
  108. package/src/store/migrations/007_sync.sql +49 -0
  109. package/src/store/migrations/008_connector_commands.sql +12 -0
  110. package/dist/dashboard/assets/chunk-5DFWNJC2.js +0 -1
  111. package/dist/dashboard/assets/chunk-S6B2HJQA.js +0 -17
@@ -71,6 +71,14 @@ export function createWrites(ctx) {
71
71
  const pendingInSessions = db.prepare(`SELECT DISTINCT e.* FROM json_each(?) s
72
72
  CROSS JOIN load_candidates c ON c.session_id = s.value AND c.ts >= ? AND c.ts < ?
73
73
  CROSS JOIN events e ON e.id = c.load_id WHERE ${pending}`);
74
+ const pendingBatch = db.prepare(`SELECT e.* FROM events e INDEXED BY events_pending_load_ts
75
+ WHERE ${pending} AND e.ts >= ? AND e.ts < ? AND (e.ts, e.rowid) > (?, ?)
76
+ ORDER BY e.ts, e.rowid LIMIT ?`);
77
+ const pendingSessionBatch = db.prepare(`SELECT DISTINCT e.rowid, e.* FROM json_each(?) s
78
+ CROSS JOIN load_candidates c ON c.session_id = s.value AND c.ts >= ? AND c.ts < ?
79
+ CROSS JOIN events e ON e.id = c.load_id
80
+ WHERE ${pending} AND (e.ts, e.rowid) > (?, ?) ORDER BY e.ts, e.rowid LIMIT ?`);
81
+ const repairCursor = db.prepare("SELECT rowid, ts, source, kind FROM events WHERE id = ?");
74
82
  const eventById = db.prepare(`SELECT *, coalesce(json_extract(data_json, '$.skillarMcp'), json_extract(data_json, '$.gisoolMcp')) AS skillarMcp,
75
83
  json_extract(data_json, '$.success') AS success FROM events WHERE id = ?`);
76
84
  const loadById = db.prepare(`SELECT e.* FROM events e WHERE e.id = ? AND ${recoverable}`);
@@ -117,16 +125,32 @@ export function createWrites(ctx) {
117
125
  .flatMap(({ value }) => loadsForTool.all(event.runtime, value, event.ts - URI_WINDOW_MS, event.ts + URI_WINDOW_MS, event.session_id));
118
126
  }
119
127
  }
128
+ else if (opts.limit !== undefined) {
129
+ const cursor = opts.after === undefined ? { ts: from, rowid: 0 }
130
+ : repairCursor.get(opts.after);
131
+ if (!cursor || cursor.ts < from || cursor.ts >= to
132
+ || ("source" in cursor && (cursor.source !== "mcp" || cursor.kind !== "load"))) {
133
+ throw storeError("invalid_repair_cursor", "repair cursor must identify an MCP load in the requested time range");
134
+ }
135
+ loads = (opts.sessionIds === undefined
136
+ ? pendingBatch.all(Math.max(from, cursor.ts), to, cursor.ts, cursor.rowid, opts.limit)
137
+ : pendingSessionBatch.all(JSON.stringify(opts.sessionIds), Math.max(from, cursor.ts), to, cursor.ts, cursor.rowid, opts.limit));
138
+ }
120
139
  else {
121
140
  loads = (opts.sessionIds === undefined ? pendingInRange.all(from, to)
122
141
  : pendingInSessions.all(JSON.stringify(opts.sessionIds), from, to));
123
142
  }
124
143
  let count = 0;
144
+ let repaired = 0;
125
145
  const uris = new Set();
126
146
  if (!opts.readOnly && loads.length > 0)
127
147
  db.exec("SAVEPOINT reconcile_loads");
128
148
  try {
129
- for (const load of loads) {
149
+ for (const selected of loads) {
150
+ // A repair CLI can run beside ingest. Read retained proof inside the transaction, not from the earlier batch.
151
+ const load = opts.limit === undefined || opts.readOnly ? selected : loadById.get(selected.id);
152
+ if (!load)
153
+ continue;
130
154
  const uri = load.skill_uri;
131
155
  if (uri === null)
132
156
  continue;
@@ -159,8 +183,11 @@ export function createWrites(ctx) {
159
183
  const json = JSON.stringify(data);
160
184
  updateLoad.run(nextId, method, reason, json, load.id, nextId, method, reason, json);
161
185
  }
162
- if (sessionId !== undefined)
186
+ if (sessionId !== undefined) {
187
+ if (load.session_id === "unattributed")
188
+ repaired += 1;
163
189
  continue;
190
+ }
164
191
  count += 1;
165
192
  uris.add(uri);
166
193
  }
@@ -172,7 +199,9 @@ export function createWrites(ctx) {
172
199
  db.exec("ROLLBACK TO reconcile_loads; RELEASE reconcile_loads");
173
200
  throw error;
174
201
  }
175
- return { count, uris: [...uris].sort() };
202
+ return { count, uris: [...uris].sort(), ...(opts.limit === undefined ? {} : {
203
+ examined: loads.length, repaired, next: loads.length === opts.limit ? loads.at(-1).id : null,
204
+ }) };
176
205
  },
177
206
  append(event, attribution) {
178
207
  const text = event.text ?? null;
@@ -212,9 +241,13 @@ export function createWrites(ctx) {
212
241
  const existing = db.prepare(`SELECT * FROM sessions WHERE runtime = ? AND (runtime_session_id IN (${marks}) OR conversation_id IN (${marks}))
213
242
  ORDER BY started_at LIMIT 1`).get(ref.runtime, ...keys, ...keys);
214
243
  if (existing) {
244
+ const values = [startedAt, ref.conversationId ?? null, ref.runtimeSessionId ?? null, ref.clientName ?? null, ref.cwd ?? null];
215
245
  db.prepare(`UPDATE sessions SET started_at = MIN(started_at, ?), conversation_id = COALESCE(conversation_id, ?),
216
- runtime_session_id = COALESCE(runtime_session_id, ?), client_name = COALESCE(client_name, ?), cwd = COALESCE(cwd, ?) WHERE id = ?`)
217
- .run(startedAt, ref.conversationId ?? null, ref.runtimeSessionId ?? null, ref.clientName ?? null, ref.cwd ?? null, existing.id);
246
+ runtime_session_id = COALESCE(runtime_session_id, ?), client_name = COALESCE(client_name, ?), cwd = COALESCE(cwd, ?)
247
+ WHERE id = ? AND (started_at IS NOT MIN(started_at, ?) OR conversation_id IS NOT COALESCE(conversation_id, ?)
248
+ OR runtime_session_id IS NOT COALESCE(runtime_session_id, ?) OR client_name IS NOT COALESCE(client_name, ?)
249
+ OR cwd IS NOT COALESCE(cwd, ?))`)
250
+ .run(...values, existing.id, ...values);
218
251
  return existing.id;
219
252
  }
220
253
  }
@@ -0,0 +1,175 @@
1
+ import { randomUUID, createHash } from "node:crypto";
2
+ import { chmodSync, mkdirSync } from "node:fs";
3
+ import { dirname } from "node:path";
4
+ import { DatabaseSync } from "node:sqlite";
5
+ import { z } from "zod";
6
+ import { CLOUD_SCHEMA } from "./cloud-schema.js";
7
+ import { acknowledgeBatch, pendingBatch } from "./outbox.js";
8
+ import { initializeQueue, spoolRequest } from "./spool-request.js";
9
+ export class CloudSyncError extends Error {
10
+ code;
11
+ constructor(code) {
12
+ super(code);
13
+ this.code = code;
14
+ this.name = "CloudSyncError";
15
+ }
16
+ }
17
+ const digest = z.string().regex(/^[a-f0-9]{64}$/);
18
+ const leaseSchema = z.object({ owner: z.string(), epoch: z.number().int(), until_ms: z.number().int() });
19
+ /** Separate durable blob DB keeps large serialization/fsync out of source commits. */
20
+ export class CloudBlobs {
21
+ maxBytes;
22
+ #db;
23
+ #owner = randomUUID();
24
+ #epoch = null;
25
+ constructor(path, maxBytes = 64 * 1024 * 1024) {
26
+ this.maxBytes = maxBytes;
27
+ z.number().int().positive().parse(maxBytes);
28
+ mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
29
+ this.#db = new DatabaseSync(path);
30
+ chmodSync(path, 0o600);
31
+ this.#db.exec(`PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL;
32
+ PRAGMA busy_timeout=5000; PRAGMA foreign_keys=ON`);
33
+ const version = z.object({ user_version: z.number() })
34
+ .parse(this.#db.prepare("PRAGMA user_version").get()).user_version;
35
+ if (version === 0) {
36
+ this.#db.exec(`BEGIN IMMEDIATE;
37
+ ${CLOUD_SCHEMA}
38
+ PRAGMA user_version=1;
39
+ COMMIT`);
40
+ }
41
+ else if (version !== 1) {
42
+ this.#db.close();
43
+ throw new CloudSyncError("unsupported_spool_version");
44
+ }
45
+ }
46
+ acquire(now = Date.now(), ttlMs = 60_000) {
47
+ z.number().int().positive().parse(ttlMs);
48
+ const row = this.#db.prepare(`UPDATE lease SET
49
+ epoch=CASE WHEN owner=? THEN epoch ELSE epoch+1 END,owner=?,until_ms=?
50
+ WHERE id=1 AND (owner=? OR until_ms<=?) RETURNING owner,epoch,until_ms`)
51
+ .get(this.#owner, this.#owner, now + ttlMs, this.#owner, now);
52
+ if (row === undefined)
53
+ return false;
54
+ this.#epoch = leaseSchema.parse(row).epoch;
55
+ return true;
56
+ }
57
+ assertLease(now = Date.now()) {
58
+ const row = leaseSchema.parse(this.#db.prepare("SELECT * FROM lease WHERE id=1").get());
59
+ if (row.owner !== this.#owner || row.epoch !== this.#epoch || row.until_ms <= now) {
60
+ throw new CloudSyncError("spool_lease_lost");
61
+ }
62
+ }
63
+ renew(now = Date.now(), ttlMs = 60_000) {
64
+ const changed = this.#db.prepare(`UPDATE lease SET until_ms=?
65
+ WHERE id=1 AND owner=? AND epoch=? AND until_ms>?`)
66
+ .run(now + ttlMs, this.#owner, this.#epoch, now);
67
+ if (changed.changes !== 1)
68
+ throw new CloudSyncError("spool_lease_lost");
69
+ }
70
+ release() {
71
+ if (this.#epoch !== null) {
72
+ this.#db.prepare("UPDATE lease SET until_ms=0 WHERE id=1 AND owner=? AND epoch=?")
73
+ .run(this.#owner, this.#epoch);
74
+ this.#epoch = null;
75
+ }
76
+ }
77
+ #transaction(run) {
78
+ this.#db.exec("BEGIN IMMEDIATE");
79
+ try {
80
+ this.assertLease();
81
+ const result = run();
82
+ this.#db.exec("COMMIT");
83
+ return result;
84
+ }
85
+ catch (error) {
86
+ this.#db.exec("ROLLBACK");
87
+ throw error;
88
+ }
89
+ }
90
+ put(bytes) {
91
+ return this.#transaction(() => {
92
+ const hash = createHash("sha256").update(bytes).digest("hex");
93
+ if (this.#db.prepare("SELECT 1 FROM blobs WHERE hash=?").get(hash) !== undefined) {
94
+ return { hash, bytes: bytes.byteLength };
95
+ }
96
+ const used = z.object({ bytes: z.number() }).parse(this.#db.prepare(`SELECT COALESCE((SELECT sum(nbytes) FROM blobs),0)+
97
+ COALESCE((SELECT sum(payload_bytes) FROM sync_outbox),0) AS bytes`).get());
98
+ if (used.bytes + bytes.byteLength > this.maxBytes)
99
+ throw new CloudSyncError("outbox_pressure");
100
+ this.#db.prepare("INSERT INTO blobs VALUES(?,?,?)").run(hash, bytes, bytes.byteLength);
101
+ return { hash, bytes: bytes.byteLength };
102
+ });
103
+ }
104
+ get(hash) {
105
+ digest.parse(hash);
106
+ const row = this.#db.prepare("SELECT bytes,nbytes FROM blobs WHERE hash=?").get(hash);
107
+ if (row === undefined)
108
+ throw new CloudSyncError("spool_blob_missing");
109
+ const saved = z.object({
110
+ bytes: z.instanceof(Uint8Array), nbytes: z.number().int().nonnegative(),
111
+ }).parse(row);
112
+ if (saved.bytes.byteLength !== saved.nbytes ||
113
+ createHash("sha256").update(saved.bytes).digest("hex") !== hash) {
114
+ throw new CloudSyncError("spool_blob_corrupt");
115
+ }
116
+ return saved.bytes;
117
+ }
118
+ prune(retained = new Set()) {
119
+ return this.#transaction(() => {
120
+ const rows = z.array(z.object({ hash: digest }))
121
+ .parse(this.#db.prepare(`SELECT hash FROM blobs b WHERE NOT EXISTS
122
+ (SELECT 1 FROM sync_outbox_blobs r WHERE r.hash=b.hash)`).all());
123
+ const remove = this.#db.prepare("DELETE FROM blobs WHERE hash=?");
124
+ let deleted = 0;
125
+ for (const row of rows)
126
+ if (!retained.has(row.hash)) {
127
+ remove.run(row.hash);
128
+ deleted++;
129
+ }
130
+ return deleted;
131
+ });
132
+ }
133
+ initialize(input) {
134
+ this.#transaction(() => initializeQueue(this.#db, input));
135
+ }
136
+ enqueue(input) {
137
+ return spoolRequest(this.#db, {
138
+ ...input,
139
+ quota: {
140
+ maxBatches: input.quota?.maxBatches ?? 1024,
141
+ maxBytes: Math.min(input.quota?.maxBytes ?? this.maxBytes, this.maxBytes),
142
+ },
143
+ assertLease: () => this.assertLease(),
144
+ storedBlobBytes: () => z.object({ bytes: z.number().int().nonnegative() })
145
+ .parse(this.#db.prepare("SELECT COALESCE(sum(nbytes),0) AS bytes FROM blobs").get()).bytes,
146
+ });
147
+ }
148
+ pending(streamId) {
149
+ return pendingBatch(this.#db, streamId);
150
+ }
151
+ pendingPrefix(prefix) {
152
+ const row = this.#db.prepare(`SELECT stream_id FROM sync_outbox
153
+ WHERE substr(stream_id,1,length(?))=? ORDER BY rowid LIMIT 1`).get(prefix, prefix);
154
+ if (row === undefined)
155
+ return null;
156
+ const { stream_id: streamId } = z.object({ stream_id: z.string() }).parse(row);
157
+ const batch = pendingBatch(this.#db, streamId);
158
+ if (batch === null)
159
+ throw new CloudSyncError("spool_sequence_gap");
160
+ return batch;
161
+ }
162
+ acknowledge(receipt) {
163
+ acknowledgeBatch(this.#db, receipt, () => this.assertLease());
164
+ }
165
+ cursor(streamId) {
166
+ const row = this.#db.prepare("SELECT ack_sequence,ack_cut FROM sync_streams WHERE stream_id=?").get(streamId);
167
+ return row === undefined ? null : z.object({
168
+ ack_sequence: z.number().int().nonnegative(), ack_cut: z.number().int().nonnegative(),
169
+ }).parse(row);
170
+ }
171
+ close() {
172
+ this.release();
173
+ this.#db.close();
174
+ }
175
+ }
@@ -0,0 +1,101 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import { z } from "zod";
3
+ import { contentKinds } from "./cloud-wire.js";
4
+ import { CloudSyncError } from "./cloud-blobs.js";
5
+ import { atomicCloudJson, currentCloudDevice, readConnection } from "./cloud-connection.js";
6
+ import { CloudHttp, CloudRequestError } from "./cloud-http.js";
7
+ import { sourceCommandsEnabled } from "./command-control.js";
8
+ import { commandPollResponseSchema, commandResultSchema, commandStatusSchema, sourceCommandSchema } from "./command-wire.js";
9
+ export const commandApplyRequestSchema = z.strictObject({
10
+ kind: z.literal("apply-command"), id: z.string().uuid(), instanceId: z.string().uuid(),
11
+ command: sourceCommandSchema,
12
+ });
13
+ const commandApplyReplySchema = z.strictObject({
14
+ kind: z.literal("command-applied"), id: z.string().uuid(), result: commandResultSchema.nullable(),
15
+ });
16
+ export function requestCommandApply(port, command, instanceId) {
17
+ if (port === null)
18
+ return Promise.resolve(null);
19
+ const id = randomUUID();
20
+ return new Promise(resolve => {
21
+ const finish = (result) => {
22
+ clearTimeout(timeout);
23
+ port.off("message", reply);
24
+ resolve(result);
25
+ };
26
+ const reply = (value) => {
27
+ const parsed = commandApplyReplySchema.safeParse(value);
28
+ if (parsed.success && parsed.data.id === id)
29
+ finish(parsed.data.result);
30
+ };
31
+ const timeout = setTimeout(() => finish(null), 5000);
32
+ port.on("message", reply);
33
+ port.postMessage({ kind: "apply-command", id, instanceId, command });
34
+ });
35
+ }
36
+ /** One bounded poll/apply/ACK in the existing sync turn; no new persistent transport. */
37
+ export async function runCloudCommandStep(home, progress, apply, options = {}) {
38
+ if (progress.phase !== "caught-up" || progress.sourceDomain === undefined || !sourceCommandsEnabled(home))
39
+ return progress;
40
+ const connection = readConnection(home);
41
+ if (connection === null || connection.instanceId !== progress.instanceId)
42
+ return progress;
43
+ const save = (state, code = null, commandId = null) => atomicCloudJson(home, "command-status.json", { state, code, commandId, updatedAtMs: Date.now() });
44
+ try {
45
+ const self = await currentCloudDevice(connection, options.fetcher);
46
+ if (!self.ingestEnabled || !self.scopes.includes("activity:read") || !self.scopes.includes("activity:ingest") ||
47
+ self.consent.mode !== "managed-content" || self.consent.scope.rootIds.length !== 0 ||
48
+ !self.consent.scope.projectIds.includes(`device:${connection.deviceId}`) ||
49
+ !contentKinds.every(kind => self.consent.mode === "managed-content" && self.consent.contentKinds.includes(kind))) {
50
+ save("disabled", "command_consent_required");
51
+ return progress;
52
+ }
53
+ const http = new CloudHttp(connection.origin, connection.token, options.fetcher);
54
+ const { command } = commandPollResponseSchema.parse(await http.request("/v1/activity/commands/poll", {
55
+ body: JSON.stringify({ sourceDomain: progress.sourceDomain }),
56
+ }));
57
+ if (command === null) {
58
+ save("idle");
59
+ return progress;
60
+ }
61
+ if (command.accountId !== connection.accountId || command.generation !== connection.generation ||
62
+ command.deviceId !== connection.deviceId || command.sourceDomain !== progress.sourceDomain ||
63
+ command.consentVersion !== self.consent.version)
64
+ throw new CloudSyncError("command_identity_mismatch");
65
+ // A disable, disconnect or consent change during poll must win before local apply.
66
+ const current = await currentCloudDevice(connection, options.fetcher);
67
+ if (!sourceCommandsEnabled(home) || readConnection(home)?.instanceId !== connection.instanceId ||
68
+ JSON.stringify(current) !== JSON.stringify(self)) {
69
+ save("disabled", "command_authority_changed", command.commandId);
70
+ return progress;
71
+ }
72
+ const result = await apply(command, connection.instanceId);
73
+ if (result === null) {
74
+ save("pending", "source_apply_unavailable", command.commandId);
75
+ return progress;
76
+ }
77
+ const receipt = commandResultSchema.parse(result);
78
+ if (receipt.commandId !== command.commandId || receipt.payloadHash !== command.payloadHash) {
79
+ throw new CloudSyncError("command_receipt_mismatch");
80
+ }
81
+ const acknowledged = commandStatusSchema.parse(await http.request("/v1/activity/commands/ack", {
82
+ body: JSON.stringify({ sourceDomain: command.sourceDomain, ...receipt }),
83
+ }));
84
+ if (acknowledged.commandId !== receipt.commandId || acknowledged.state !== receipt.state ||
85
+ acknowledged.sourceRevision !== receipt.sourceRevision)
86
+ throw new CloudSyncError("command_ack_mismatch");
87
+ save(receipt.state, receipt.error, command.commandId);
88
+ return progress;
89
+ }
90
+ catch (failure) {
91
+ if (failure instanceof CloudRequestError) {
92
+ save(failure.state, failure.code);
93
+ return progress;
94
+ }
95
+ if (failure instanceof CloudSyncError) {
96
+ save("quarantined", failure.code);
97
+ return progress;
98
+ }
99
+ throw failure;
100
+ }
101
+ }
@@ -0,0 +1,105 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeFileSync, } from "node:fs";
3
+ import { join } from "node:path";
4
+ import { z } from "zod";
5
+ import { CloudSyncError } from "./cloud-blobs.js";
6
+ import { CloudHttp, cloudOriginSchema } from "./cloud-http.js";
7
+ import { consentSchema } from "./cloud-wire.js";
8
+ export const deviceSelfSchema = z.strictObject({
9
+ accountId: z.string().min(1), deviceId: z.string().min(1),
10
+ generation: z.number().int().positive(), shardId: z.literal("activity-00"),
11
+ scopes: z.array(z.enum(["activity:read", "activity:ingest", "skills:read", "skills:write"])).min(1),
12
+ ingestEnabled: z.boolean(), version: z.number().int().nonnegative(),
13
+ consent: consentSchema,
14
+ });
15
+ export const connectionSchema = z.strictObject({
16
+ version: z.literal(1), instanceId: z.string().uuid(), origin: cloudOriginSchema,
17
+ accountId: z.string().min(1), deviceId: z.string().min(1),
18
+ generation: z.number().int().positive(), token: z.string().regex(/^[A-Za-z0-9_-]{43}$/),
19
+ expiresAt: z.number().int().positive(),
20
+ // Last scopes the server confirmed for this device; absent in files written
21
+ // before scopes were persisted, where the caller probes or assumes instead.
22
+ scopes: z.array(z.enum(["activity:read", "activity:ingest", "skills:read", "skills:write"]))
23
+ .min(1).optional(),
24
+ });
25
+ export function cloudHome(home) {
26
+ return join(home, "cloud");
27
+ }
28
+ export function atomicCloudJson(home, name, value) {
29
+ if (!/^[a-z0-9-]+\.json$/.test(name))
30
+ throw new CloudSyncError("invalid_state_path");
31
+ const directory = cloudHome(home);
32
+ mkdirSync(directory, { recursive: true, mode: 0o700 });
33
+ const target = join(directory, name), temporary = `${target}.${randomUUID()}.tmp`;
34
+ const fd = openSync(temporary, "wx", 0o600);
35
+ try {
36
+ writeFileSync(fd, JSON.stringify(value));
37
+ fsyncSync(fd);
38
+ }
39
+ finally {
40
+ closeSync(fd);
41
+ }
42
+ renameSync(temporary, target);
43
+ const parent = openSync(directory, "r");
44
+ try {
45
+ fsyncSync(parent);
46
+ }
47
+ finally {
48
+ closeSync(parent);
49
+ }
50
+ }
51
+ export function readConnection(home) {
52
+ const path = join(cloudHome(home), "connection.json");
53
+ if (!existsSync(path))
54
+ return null;
55
+ const input = JSON.parse(readFileSync(path, "utf8"));
56
+ return connectionSchema.parse(input);
57
+ }
58
+ export function disconnectCloud(home) {
59
+ const path = join(cloudHome(home), "connection.json");
60
+ if (existsSync(path))
61
+ unlinkSync(path);
62
+ // Source journal, immutable pending requests, blobs and local backups survive.
63
+ }
64
+ /** Connection exchanges a browser-issued grant; it never grants upload consent. */
65
+ export async function connectCloud(home, input, fetcher = fetch) {
66
+ const existing = readConnection(home);
67
+ if (existing !== null &&
68
+ (input.replaceDeviceId === undefined || existing.deviceId !== input.replaceDeviceId)) {
69
+ throw new CloudSyncError("already_connected");
70
+ }
71
+ const exchange = new CloudHttp(input.origin, null, fetcher);
72
+ const credential = z.strictObject({
73
+ token: z.string().regex(/^[A-Za-z0-9_-]{43}$/), deviceId: z.string().min(1),
74
+ expiresAt: z.number().int().positive(),
75
+ }).parse(await exchange.request("/v1/devices/exchange", {
76
+ body: JSON.stringify({ grant: z.string().min(32).max(128).parse(input.grant) }),
77
+ }));
78
+ const self = deviceSelfSchema.parse(await new CloudHttp(input.origin, credential.token, fetcher)
79
+ .request("/v1/devices/self"));
80
+ if (self.deviceId !== credential.deviceId)
81
+ throw new CloudSyncError("device_identity_mismatch");
82
+ // A scope upgrade must land on the device it was approved for, nothing else.
83
+ if (input.replaceDeviceId !== undefined && credential.deviceId !== input.replaceDeviceId) {
84
+ throw new CloudSyncError("device_identity_mismatch");
85
+ }
86
+ const connection = connectionSchema.parse({
87
+ version: 1, instanceId: randomUUID(), origin: input.origin, accountId: self.accountId,
88
+ deviceId: self.deviceId, generation: self.generation,
89
+ token: credential.token, expiresAt: credential.expiresAt, scopes: [...self.scopes],
90
+ });
91
+ atomicCloudJson(home, "connection.json", connection);
92
+ const { token: _secret, ...visible } = connection;
93
+ return visible;
94
+ }
95
+ export async function currentCloudDevice(connection, fetcher = fetch, attempt = 0) {
96
+ if (connection.expiresAt <= Math.floor(Date.now() / 1000)) {
97
+ throw new CloudSyncError("credentials_expired");
98
+ }
99
+ const self = deviceSelfSchema.parse(await new CloudHttp(connection.origin, connection.token, fetcher, 30_000, attempt)
100
+ .request("/v1/devices/self"));
101
+ if (self.deviceId !== connection.deviceId || self.accountId !== connection.accountId ||
102
+ self.generation !== connection.generation)
103
+ throw new CloudSyncError("device_identity_mismatch");
104
+ return self;
105
+ }
@@ -0,0 +1,114 @@
1
+ import { z } from "zod";
2
+ import { CloudSyncError } from "./cloud-blobs.js";
3
+ import { sha256 } from "./cloud-documents.js";
4
+ import { CloudRequestError } from "./cloud-http.js";
5
+ import { requireJob } from "./cloud-jobs.js";
6
+ import { contentChunkRefSchema, migrationPageSchema, migrationStatusSchema, parseContentBatch, parseContentChunkUpload, } from "./cloud-wire.js";
7
+ export const receiptSchema = z.strictObject({
8
+ batchId: z.string(), payloadHash: z.string().regex(/^[a-f0-9]{64}$/),
9
+ sequence: z.number().int().positive(), durableAt: z.number().int(),
10
+ projectedThrough: z.number().int().nonnegative(), changeSeq: z.number().int().positive(),
11
+ state: z.literal("projected"),
12
+ });
13
+ async function request(blobs, http, path, body) {
14
+ blobs.renew();
15
+ const value = await http.request(path, { body });
16
+ blobs.renew();
17
+ return value;
18
+ }
19
+ async function upload(blobs, http, refs, expected) {
20
+ if (refs.length !== expected.length)
21
+ throw new CloudSyncError("spool_blob_corrupt");
22
+ for (const [index, reference] of refs.entries()) {
23
+ const bytes = blobs.get(reference.hash);
24
+ if (bytes.byteLength !== reference.bytes)
25
+ throw new CloudSyncError("spool_blob_corrupt");
26
+ const body = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
27
+ const chunk = parseContentChunkUpload(JSON.parse(body));
28
+ const declared = contentChunkRefSchema.parse({
29
+ index: chunk.index, revision: chunk.revision,
30
+ plaintextHash: chunk.plaintextHash, bytes: chunk.bytes,
31
+ });
32
+ // The strict reference parser rejects extra chunk fields; select only the
33
+ // machine-consumed identity before comparing the server's durable chunk ACK.
34
+ const wanted = expected[index];
35
+ const ack = z.strictObject({ stored: z.literal(true), chunk: contentChunkRefSchema })
36
+ .parse(await request(blobs, http, "/v1/activity/chunks", body));
37
+ if (wanted === undefined || JSON.stringify(ack.chunk) !== JSON.stringify(wanted) ||
38
+ declared.index !== wanted.index || declared.plaintextHash !== wanted.plaintextHash ||
39
+ declared.bytes !== wanted.bytes || declared.revision !== wanted.revision) {
40
+ throw new CloudSyncError("invalid_remote_receipt");
41
+ }
42
+ }
43
+ }
44
+ /** A bounded unit of work; an accepted snapshot is never inferred from a page ACK. */
45
+ export async function deliverCloudJob(blobs, http, batch) {
46
+ const job = requireJob(batch.payload);
47
+ switch (job.kind) {
48
+ case "cdc": {
49
+ const body = parseContentBatch(JSON.parse(job.body));
50
+ if (body.sequence !== batch.sequence || body.batchId !== batch.batch_id ||
51
+ body.journal.from !== batch.journal_from || body.journal.through !== batch.journal_through ||
52
+ sha256(JSON.stringify(body)) !== job.payloadHash)
53
+ throw new CloudSyncError("invalid_spool_job");
54
+ try {
55
+ await upload(blobs, http, job.refs, body.chunks);
56
+ }
57
+ catch (error) {
58
+ // Expired chunks can belong to an accepted batch whose ACK was lost.
59
+ // Only the exact receipt below can acknowledge it; otherwise stay pending.
60
+ if (!(error instanceof CloudRequestError && error.status === 409 && error.code === "content_expired")) {
61
+ throw error;
62
+ }
63
+ }
64
+ const receipt = receiptSchema.parse(await request(blobs, http, "/v1/activity/batches", job.body));
65
+ if (receipt.batchId !== body.batchId || receipt.sequence !== body.sequence ||
66
+ receipt.payloadHash !== job.payloadHash || receipt.projectedThrough !== body.journal.through) {
67
+ throw new CloudSyncError("invalid_remote_receipt");
68
+ }
69
+ return { accepted: true, kind: "cdc", excluded: job.excluded, receipt };
70
+ }
71
+ case "migration": {
72
+ const status = migrationStatusSchema.parse(await request(blobs, http, "/v1/activity/migrations/begin", JSON.stringify(job.begin)));
73
+ if (JSON.stringify(status.migration) !== JSON.stringify(job.begin)) {
74
+ throw new CloudSyncError("invalid_remote_receipt");
75
+ }
76
+ if (status.state === "abandoned")
77
+ throw new CloudSyncError("migration_closed");
78
+ const reference = {
79
+ deviceId: job.begin.deviceId, generation: job.begin.generation,
80
+ sourceDomain: job.begin.sourceDomain, snapshotId: job.begin.snapshotId,
81
+ };
82
+ const staged = new Map(status.stagedPages.map(page => [page.index, page]));
83
+ for (const page of job.pages) {
84
+ const previous = staged.get(page.page.index);
85
+ if (previous !== undefined && JSON.stringify(previous) !== JSON.stringify(page.page)) {
86
+ throw new CloudSyncError("invalid_remote_receipt");
87
+ }
88
+ }
89
+ const next = job.pages.find(page => !staged.has(page.page.index));
90
+ if (next !== undefined) {
91
+ await upload(blobs, http, next.refs, next.page.chunks);
92
+ const saved = z.strictObject({ staged: z.literal(true), page: migrationPageSchema })
93
+ .parse(await request(blobs, http, "/v1/activity/migrations/page", JSON.stringify(next.page)));
94
+ if (JSON.stringify(saved.page) !== JSON.stringify(next.page)) {
95
+ throw new CloudSyncError("invalid_remote_receipt");
96
+ }
97
+ return { accepted: false, kind: "migration", excluded: job.excluded,
98
+ stagedPages: staged.size + 1, totalPages: job.pages.length };
99
+ }
100
+ const completed = status.accepted ? status : migrationStatusSchema.parse(await request(blobs, http, "/v1/activity/migrations/complete", JSON.stringify(reference)));
101
+ if (!completed.accepted || JSON.stringify(completed.migration) !== JSON.stringify(job.begin) ||
102
+ (job.begin.purpose === "initial" &&
103
+ (completed.cursor.sequence !== 0 || completed.cursor.journalThrough !== job.begin.cut))) {
104
+ throw new CloudSyncError("invalid_remote_receipt");
105
+ }
106
+ return { accepted: true, kind: "migration", excluded: job.excluded,
107
+ status: completed };
108
+ }
109
+ default: {
110
+ const exhaustive = job;
111
+ return exhaustive;
112
+ }
113
+ }
114
+ }