skillar 0.2.1 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +280 -0
- package/README.md +37 -5
- package/dist/analysis/finding.js +8 -190
- package/dist/analysis/kernel.js +896 -0
- package/dist/analysis/tree.js +7 -154
- package/dist/analysis/usage.js +9 -394
- package/dist/api/flows.js +42 -212
- package/dist/api/report-kernel.js +1063 -0
- package/dist/api/routes.js +12 -2
- package/dist/catalog/fields.js +138 -146
- package/dist/catalog/watcher.js +7 -2
- package/dist/cli/init.js +45 -0
- package/dist/cli/main.js +76 -3
- package/dist/cli/private-skills.js +122 -0
- package/dist/cli/serve.js +15 -1
- package/dist/cli/sync.js +124 -0
- package/dist/dashboard/NOTICE +280 -0
- package/dist/dashboard/assets/app.js +737 -275
- package/dist/dashboard/assets/astryx-reset.css +407 -0
- package/dist/dashboard/assets/astryx.css +1077 -0
- package/dist/dashboard/assets/charts.css +54 -0
- package/dist/dashboard/assets/chunk-TPGDA2J4.js +1 -0
- package/dist/dashboard/assets/chunk-YLWHVDMP.js +36 -0
- package/dist/dashboard/assets/controls.css +207 -0
- package/dist/dashboard/assets/overlays.css +132 -0
- package/dist/dashboard/assets/p1.css +135 -0
- package/dist/dashboard/assets/shell.css +71 -0
- package/dist/dashboard/assets/skillar-theme.css +692 -0
- package/dist/dashboard/assets/styles.css +224 -121
- package/dist/dashboard/assets/tokens.css +278 -0
- package/dist/dashboard/assets/widgets.css +83 -0
- package/dist/dashboard/index.html +11 -2
- package/dist/dashboard/viewers/viewers.css +61 -0
- package/dist/ingest/attribution.js +5 -1
- package/dist/ingest/pipeline.js +2 -0
- package/dist/ingest/request-recorder.js +39 -0
- package/dist/mcp/dispatch.js +2 -2
- package/dist/mcp/http.js +43 -3
- package/dist/mcp/skills-extension.js +42 -15
- package/dist/mcp/stdio-proxy.js +24 -3
- package/dist/mcp/tools.js +38 -24
- package/dist/packs/claude-code.js +22 -11
- package/dist/packs/cursor.js +9 -4
- package/dist/packs/gemini.js +9 -4
- package/dist/packs/json-config.js +41 -0
- package/dist/packs/omo.js +15 -6
- package/dist/request-status.js +81 -0
- package/dist/search/fts.js +93 -58
- package/dist/search/fusion.js +49 -47
- package/dist/search/guidance.js +18 -6
- package/dist/search/index.js +291 -70
- package/dist/search/lexical.js +71 -63
- package/dist/server.js +12 -0
- package/dist/skills/apply.js +130 -0
- package/dist/skills/capture.js +54 -0
- package/dist/skills/cloud-plan.js +57 -0
- package/dist/skills/cloud.js +81 -0
- package/dist/skills/fs.js +133 -0
- package/dist/skills/package.js +734 -0
- package/dist/skills/plan.js +83 -0
- package/dist/skills/receipt.js +44 -0
- package/dist/skills/rollback.js +119 -0
- package/dist/skills/transfer.js +51 -0
- package/dist/store/db.js +7 -1
- package/dist/store/execution.js +2 -34
- package/dist/store/export.js +25 -4
- package/dist/store/migrations/007_sync.sql +49 -0
- package/dist/store/migrations/008_connector_commands.sql +12 -0
- package/dist/store/queries.js +33 -7
- package/dist/store/writes.js +38 -5
- package/dist/sync/cloud-blobs.js +175 -0
- package/dist/sync/cloud-commands.js +101 -0
- package/dist/sync/cloud-connection.js +105 -0
- package/dist/sync/cloud-delivery.js +114 -0
- package/dist/sync/cloud-documents.js +103 -0
- package/dist/sync/cloud-export.js +67 -0
- package/dist/sync/cloud-http.js +150 -0
- package/dist/sync/cloud-jobs.js +94 -0
- package/dist/sync/cloud-pending.js +143 -0
- package/dist/sync/cloud-prepare.js +101 -0
- package/dist/sync/cloud-restore.js +136 -0
- package/dist/sync/cloud-rollback.js +24 -0
- package/dist/sync/cloud-runner.js +86 -0
- package/dist/sync/cloud-runtime.js +293 -0
- package/dist/sync/cloud-schema.js +32 -0
- package/dist/sync/cloud-stage.js +112 -0
- package/dist/sync/cloud-wire.js +693 -0
- package/dist/sync/cloud-worker.js +57 -0
- package/dist/sync/cloud.js +157 -0
- package/dist/sync/command-apply.js +140 -0
- package/dist/sync/command-control.js +25 -0
- package/dist/sync/command-wire.js +437 -0
- package/dist/sync/consent.js +130 -0
- package/dist/sync/journal.js +43 -0
- package/dist/sync/outbox.js +134 -0
- package/dist/sync/restore-rows.js +125 -0
- package/dist/sync/restore.js +174 -0
- package/dist/sync/snapshot.js +78 -0
- package/dist/sync/spool-request.js +103 -0
- package/dist/sync/tables.js +27 -0
- package/dist/sync/triggers.js +34 -0
- package/dist/workflow/compare.js +3 -43
- package/docs/cloud-sync.md +94 -0
- package/docs/private-skills.md +75 -0
- package/docs/sync-p4.md +102 -0
- package/docs/sync-restore.md +167 -0
- package/package.json +18 -5
- package/src/store/migrations/007_sync.sql +49 -0
- package/src/store/migrations/008_connector_commands.sql +12 -0
- package/dist/dashboard/assets/chunk-5DFWNJC2.js +0 -1
- package/dist/dashboard/assets/chunk-S6B2HJQA.js +0 -17
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { closeSync, existsSync, fsyncSync, linkSync, openSync, unlinkSync } from "node:fs";
|
|
3
|
+
import { dirname } from "node:path";
|
|
4
|
+
import { openDb } from "../store/db.js";
|
|
5
|
+
import { CloudSyncError } from "./cloud-blobs.js";
|
|
6
|
+
import { readConnection } from "./cloud-connection.js";
|
|
7
|
+
import { sha256 } from "./cloud-documents.js";
|
|
8
|
+
import { downloadSnapshot, exportPageSchema, verifyExportBatch } from "./cloud-export.js";
|
|
9
|
+
import { CloudHttp } from "./cloud-http.js";
|
|
10
|
+
import { cloudScope } from "./cloud-stage.js";
|
|
11
|
+
import { currentCloudDevice } from "./cloud-connection.js";
|
|
12
|
+
import { applyCatchup, restoreSnapshot, verify } from "./restore.js";
|
|
13
|
+
import { identity, readRows } from "./restore-rows.js";
|
|
14
|
+
import { mutationSchema } from "./tables.js";
|
|
15
|
+
import { migrationReferenceSchema, summarizeSnapshot } from "./cloud-wire.js";
|
|
16
|
+
const asMutation = (row) => mutationSchema.parse({ ...row, beforeValues: null });
|
|
17
|
+
const asContent = ({ beforeValues: _before, ...row }) => row;
|
|
18
|
+
async function verifyDataset(db, rows, cut) {
|
|
19
|
+
const saved = db.prepare("SELECT clock_json FROM sync_restore_state WHERE singleton=1").get();
|
|
20
|
+
const clock = mutationSchema.parse(JSON.parse(String(saved?.["clock_json"])));
|
|
21
|
+
const actual = await summarizeSnapshot(readRows(db, cut, clock).map(asContent));
|
|
22
|
+
const expected = await summarizeSnapshot(rows.map(row => ({ ...row, revision: cut })));
|
|
23
|
+
if (actual.snapshotHash !== expected.snapshotHash ||
|
|
24
|
+
JSON.stringify(actual.counts) !== JSON.stringify(expected.counts)) {
|
|
25
|
+
throw new CloudSyncError("restore_integrity");
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
/** New destination only. Publish a fully verified DB atomically without replacing a file. */
|
|
29
|
+
export async function restoreCloud(home, destination, input, fetcher = fetch) {
|
|
30
|
+
if (existsSync(destination))
|
|
31
|
+
throw new CloudSyncError("destination_exists");
|
|
32
|
+
const connection = readConnection(home);
|
|
33
|
+
if (connection === null)
|
|
34
|
+
throw new CloudSyncError("not_connected");
|
|
35
|
+
const reference = migrationReferenceSchema.parse(input);
|
|
36
|
+
if (reference.deviceId !== connection.deviceId || reference.generation !== connection.generation) {
|
|
37
|
+
throw new CloudSyncError("device_identity_mismatch");
|
|
38
|
+
}
|
|
39
|
+
const self = await currentCloudDevice(connection, fetcher);
|
|
40
|
+
const scope = cloudScope(connection, self.consent, self.consent.mode === "managed-content" &&
|
|
41
|
+
self.consent.backfill !== undefined);
|
|
42
|
+
if (scope.contentKinds.length !== 7)
|
|
43
|
+
throw new CloudSyncError("full_snapshot_required");
|
|
44
|
+
const http = new CloudHttp(connection.origin, connection.token, fetcher);
|
|
45
|
+
const { manifest, rows } = await downloadSnapshot(http, reference);
|
|
46
|
+
const cut = manifest.migration.cut;
|
|
47
|
+
const records = rows.map(asMutation), payload = JSON.stringify(records);
|
|
48
|
+
const temporary = `${destination}.${randomUUID()}.tmp`;
|
|
49
|
+
closeSync(openSync(temporary, "wx", 0o600));
|
|
50
|
+
let db;
|
|
51
|
+
try {
|
|
52
|
+
db = openDb(temporary);
|
|
53
|
+
restoreSnapshot(db, { sourceDomain: reference.sourceDomain, cut, counts: manifest.migration.counts }, [{
|
|
54
|
+
snapshotId: reference.snapshotId, sourceDomain: reference.sourceDomain, cut,
|
|
55
|
+
index: 0, records, bytes: Buffer.byteLength(payload), hash: sha256(payload),
|
|
56
|
+
}]);
|
|
57
|
+
await verifyDataset(db, rows, cut);
|
|
58
|
+
const dataset = new Map(records.map(row => [identity(row), asContent(row)]));
|
|
59
|
+
let afterSequence = 0, throughSequence, currentCut = cut;
|
|
60
|
+
for (;;) {
|
|
61
|
+
const page = exportPageSchema.parse(await http.request("/v1/activity/export", {
|
|
62
|
+
body: JSON.stringify({
|
|
63
|
+
...reference, snapshotId: undefined, consentVersion: scope.consentVersion,
|
|
64
|
+
projectId: scope.projectId, rootId: scope.rootId,
|
|
65
|
+
range: { fromMs: scope.fromMs, toMs: scope.toMs }, backfill: scope.backfill,
|
|
66
|
+
afterSequence, throughSequence, limit: 1,
|
|
67
|
+
}), maxResponseBytes: 18 * 1024 * 1024,
|
|
68
|
+
}));
|
|
69
|
+
if (page.deviceId !== reference.deviceId || page.sourceDomain !== reference.sourceDomain ||
|
|
70
|
+
page.generation !== reference.generation || page.afterSequence !== afterSequence ||
|
|
71
|
+
(throughSequence !== undefined && page.throughSequence !== throughSequence) ||
|
|
72
|
+
page.start?.sequence !== afterSequence || page.start.journalThrough > currentCut) {
|
|
73
|
+
throw new CloudSyncError("restore_history_gap");
|
|
74
|
+
}
|
|
75
|
+
throughSequence ??= page.throughSequence;
|
|
76
|
+
let previousCut = page.start.journalThrough;
|
|
77
|
+
for (const item of page.batches) {
|
|
78
|
+
verifyExportBatch(item);
|
|
79
|
+
const batch = item.batch;
|
|
80
|
+
if (batch.sequence !== afterSequence + 1 || batch.journal.from !== previousCut ||
|
|
81
|
+
batch.deviceId !== reference.deviceId || batch.sourceDomain !== reference.sourceDomain ||
|
|
82
|
+
batch.generation !== reference.generation)
|
|
83
|
+
throw new CloudSyncError("restore_history_gap");
|
|
84
|
+
afterSequence = batch.sequence;
|
|
85
|
+
previousCut = batch.journal.through;
|
|
86
|
+
if (previousCut <= currentCut)
|
|
87
|
+
continue;
|
|
88
|
+
const changes = item.document.rows.filter(row => row.revision > currentCut).map(asMutation);
|
|
89
|
+
for (const row of changes) {
|
|
90
|
+
if (row.operation === "delete")
|
|
91
|
+
dataset.delete(identity(row));
|
|
92
|
+
else
|
|
93
|
+
dataset.set(identity(row), asContent(row));
|
|
94
|
+
}
|
|
95
|
+
applyCatchup(db, {
|
|
96
|
+
sourceDomain: reference.sourceDomain, after: currentCut, cut: previousCut, records: changes,
|
|
97
|
+
});
|
|
98
|
+
currentCut = previousCut;
|
|
99
|
+
await verifyDataset(db, [...dataset.values()], currentCut);
|
|
100
|
+
}
|
|
101
|
+
if (page.cursor?.sequence !== afterSequence || page.cursor.journalThrough !== previousCut ||
|
|
102
|
+
page.hasMore !== (afterSequence < throughSequence) ||
|
|
103
|
+
(page.hasMore && page.batches.length === 0))
|
|
104
|
+
throw new CloudSyncError("restore_history_gap");
|
|
105
|
+
if (!page.hasMore)
|
|
106
|
+
break;
|
|
107
|
+
}
|
|
108
|
+
const result = verify(db, { sourceDomain: reference.sourceDomain, cut: currentCut });
|
|
109
|
+
db.exec("PRAGMA wal_checkpoint(TRUNCATE)");
|
|
110
|
+
db.close();
|
|
111
|
+
db = undefined;
|
|
112
|
+
const fd = openSync(temporary, "r");
|
|
113
|
+
try {
|
|
114
|
+
fsyncSync(fd);
|
|
115
|
+
}
|
|
116
|
+
finally {
|
|
117
|
+
closeSync(fd);
|
|
118
|
+
}
|
|
119
|
+
linkSync(temporary, destination);
|
|
120
|
+
const parent = openSync(dirname(destination), "r");
|
|
121
|
+
try {
|
|
122
|
+
fsyncSync(parent);
|
|
123
|
+
}
|
|
124
|
+
finally {
|
|
125
|
+
closeSync(parent);
|
|
126
|
+
}
|
|
127
|
+
return { ...result, throughSequence };
|
|
128
|
+
}
|
|
129
|
+
finally {
|
|
130
|
+
db?.close();
|
|
131
|
+
for (const path of [temporary, `${temporary}-wal`, `${temporary}-shm`]) {
|
|
132
|
+
if (existsSync(path))
|
|
133
|
+
unlinkSync(path);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { CloudSyncError } from "./cloud-blobs.js";
|
|
4
|
+
import { cloudHome, readConnection } from "./cloud-connection.js";
|
|
5
|
+
import { CloudHttp } from "./cloud-http.js";
|
|
6
|
+
import { preparedSchema } from "./cloud-prepare.js";
|
|
7
|
+
import { pauseCloud } from "./cloud.js";
|
|
8
|
+
import { migrationStatusSchema } from "./cloud-wire.js";
|
|
9
|
+
/** Rollback pauses delivery; accepted cloud history and local capture remain intact. */
|
|
10
|
+
export async function rollbackCloud(home, fetcher = fetch) {
|
|
11
|
+
pauseCloud(home);
|
|
12
|
+
const connection = readConnection(home);
|
|
13
|
+
if (connection === null)
|
|
14
|
+
throw new CloudSyncError("not_connected");
|
|
15
|
+
const prepared = preparedSchema.parse(JSON.parse(readFileSync(join(cloudHome(home), "prepared.json"), "utf8")));
|
|
16
|
+
if (prepared.instanceId !== connection.instanceId || prepared.deviceId !== connection.deviceId ||
|
|
17
|
+
prepared.generation !== connection.generation)
|
|
18
|
+
throw new CloudSyncError("prepared_snapshot_mismatch");
|
|
19
|
+
return migrationStatusSchema.parse(await new CloudHttp(connection.origin, connection.token, fetcher)
|
|
20
|
+
.request("/v1/activity/migrations/rollback", { body: JSON.stringify({
|
|
21
|
+
deviceId: prepared.deviceId, generation: prepared.generation,
|
|
22
|
+
sourceDomain: prepared.sourceDomain, snapshotId: prepared.snapshotId,
|
|
23
|
+
}) }));
|
|
24
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { existsSync, watch } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { Worker } from "node:worker_threads";
|
|
4
|
+
import { cloudHome } from "./cloud-connection.js";
|
|
5
|
+
import { cloudProgressSchema } from "./cloud.js";
|
|
6
|
+
import { runtimeCaptureRequestSchema } from "./cloud-runtime.js";
|
|
7
|
+
import { commandApplyRequestSchema } from "./cloud-commands.js";
|
|
8
|
+
/** No worker or network connection exists until a browser-authorized credential does. */
|
|
9
|
+
export function startCloudSync(home, source, log, captureRuntime, applyCommand) {
|
|
10
|
+
let worker, controls, closed = false;
|
|
11
|
+
let lastPhase;
|
|
12
|
+
const reconcile = () => {
|
|
13
|
+
if (closed)
|
|
14
|
+
return;
|
|
15
|
+
const directory = cloudHome(home);
|
|
16
|
+
if (controls === undefined && existsSync(directory)) {
|
|
17
|
+
controls = watch(directory, (_event, filename) => {
|
|
18
|
+
if (filename === "connection.json")
|
|
19
|
+
reconcile();
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
if (!existsSync(join(directory, "connection.json"))) {
|
|
23
|
+
if (worker !== undefined)
|
|
24
|
+
void worker.terminate();
|
|
25
|
+
worker = undefined;
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
if (worker !== undefined) {
|
|
29
|
+
worker.postMessage("wake");
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
const started = new Worker(new URL("./cloud-worker.js", import.meta.url), {
|
|
33
|
+
workerData: { home, source, captureRuntime: captureRuntime !== undefined, applyCommands: applyCommand !== undefined },
|
|
34
|
+
});
|
|
35
|
+
worker = started;
|
|
36
|
+
started.on("message", value => {
|
|
37
|
+
const command = commandApplyRequestSchema.safeParse(value);
|
|
38
|
+
if (command.success) {
|
|
39
|
+
if (closed || worker !== started)
|
|
40
|
+
return;
|
|
41
|
+
let result = null;
|
|
42
|
+
try {
|
|
43
|
+
result = applyCommand?.(command.data.command, command.data.instanceId) ?? null;
|
|
44
|
+
}
|
|
45
|
+
catch (failure) {
|
|
46
|
+
// RPC boundary: never expose source content or credential details.
|
|
47
|
+
if (!(failure instanceof Error))
|
|
48
|
+
throw failure;
|
|
49
|
+
log("cloud command: source_apply_failed");
|
|
50
|
+
}
|
|
51
|
+
started.postMessage({ kind: "command-applied", id: command.data.id, result });
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
const request = runtimeCaptureRequestSchema.safeParse(value);
|
|
55
|
+
if (request.success) {
|
|
56
|
+
if (closed || worker !== started)
|
|
57
|
+
return;
|
|
58
|
+
started.postMessage({
|
|
59
|
+
kind: "runtime-captured", id: request.data.id, capture: captureRuntime?.() ?? null,
|
|
60
|
+
});
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
const progress = cloudProgressSchema.safeParse(value);
|
|
64
|
+
if (progress.success && progress.data.phase !== lastPhase) {
|
|
65
|
+
lastPhase = progress.data.phase;
|
|
66
|
+
log(`cloud sync: ${lastPhase}`);
|
|
67
|
+
}
|
|
68
|
+
});
|
|
69
|
+
started.on("error", () => log("cloud sync: worker_failed"));
|
|
70
|
+
started.on("exit", () => { if (worker === started)
|
|
71
|
+
worker = undefined; });
|
|
72
|
+
};
|
|
73
|
+
const root = watch(home, (_event, filename) => { if (filename === "cloud")
|
|
74
|
+
reconcile(); });
|
|
75
|
+
reconcile();
|
|
76
|
+
return {
|
|
77
|
+
close() {
|
|
78
|
+
closed = true;
|
|
79
|
+
root.close();
|
|
80
|
+
controls?.close();
|
|
81
|
+
if (worker !== undefined)
|
|
82
|
+
void worker.terminate();
|
|
83
|
+
worker = undefined;
|
|
84
|
+
},
|
|
85
|
+
};
|
|
86
|
+
}
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
import { activeRequests } from "../request-status.js";
|
|
6
|
+
import { openDb } from "../store/db.js";
|
|
7
|
+
import { readIngestCounters } from "../store/writes.js";
|
|
8
|
+
import { CloudBlobs, CloudSyncError } from "./cloud-blobs.js";
|
|
9
|
+
import { atomicCloudJson, cloudHome, currentCloudDevice, readConnection } from "./cloud-connection.js";
|
|
10
|
+
import { blobReferenceSchema, sha256 } from "./cloud-documents.js";
|
|
11
|
+
import { CloudHttp, CloudRequestError } from "./cloud-http.js";
|
|
12
|
+
import { cloudScope } from "./cloud-stage.js";
|
|
13
|
+
import { journalHead } from "./journal.js";
|
|
14
|
+
import { SyncError } from "./tables.js";
|
|
15
|
+
import { ActivityContractError, analysisRuntimeSchema, authorizeUpload, contentChunkRefSchema, contentKinds, MAX_CONTENT_CHUNK_BYTES, parseContentChunkUpload, runtimeCommitSchema, } from "./cloud-wire.js";
|
|
16
|
+
/** Called only by the daemon's main-thread RPC handler, never by the sync worker. */
|
|
17
|
+
export function captureDaemonRuntime(db) {
|
|
18
|
+
const head = journalHead(db);
|
|
19
|
+
return {
|
|
20
|
+
sourceDomain: head.sourceDomain, sourceCut: head.cut,
|
|
21
|
+
context: {
|
|
22
|
+
schemaVersion: 1, capturedAtMs: Date.now(), activeRequests: [...activeRequests],
|
|
23
|
+
ingest: readIngestCounters(db), tz: Intl.DateTimeFormat().resolvedOptions().timeZone,
|
|
24
|
+
},
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
export const runtimeCaptureRequestSchema = z.strictObject({
|
|
28
|
+
kind: z.literal("capture-runtime"), id: z.string().uuid(),
|
|
29
|
+
sourceDomain: z.string(), sourceCut: z.number().int().nonnegative(),
|
|
30
|
+
});
|
|
31
|
+
export const runtimeCaptureReplySchema = z.strictObject({
|
|
32
|
+
kind: z.literal("runtime-captured"), id: z.string().uuid(),
|
|
33
|
+
capture: z.strictObject({
|
|
34
|
+
sourceDomain: z.string(), sourceCut: z.number().int().nonnegative(), context: analysisRuntimeSchema,
|
|
35
|
+
}).nullable(),
|
|
36
|
+
});
|
|
37
|
+
export function requestRuntimeCapture(port, request) {
|
|
38
|
+
if (port === null)
|
|
39
|
+
return Promise.resolve(null);
|
|
40
|
+
// Each request owns its listener, correlation ID and timeout.
|
|
41
|
+
return new Promise(resolve => {
|
|
42
|
+
const finish = (value) => {
|
|
43
|
+
clearTimeout(timeout);
|
|
44
|
+
port.off("message", reply);
|
|
45
|
+
resolve(value);
|
|
46
|
+
};
|
|
47
|
+
const reply = (value) => {
|
|
48
|
+
const parsed = runtimeCaptureReplySchema.safeParse(value);
|
|
49
|
+
if (parsed.success && parsed.data.id === request.id)
|
|
50
|
+
finish(parsed.data.capture);
|
|
51
|
+
};
|
|
52
|
+
const timeout = setTimeout(() => finish(null), 5000);
|
|
53
|
+
port.on("message", reply);
|
|
54
|
+
port.postMessage(request);
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
const jobSchema = z.strictObject({
|
|
58
|
+
kind: z.literal("runtime"), body: z.string(), refs: z.array(blobReferenceSchema),
|
|
59
|
+
fingerprint: z.string().regex(/^[a-f0-9]{64}$/),
|
|
60
|
+
preparedAtMs: z.number().int().optional(),
|
|
61
|
+
});
|
|
62
|
+
const receiptSchema = z.strictObject({
|
|
63
|
+
sequence: z.number().int().positive(), sourceCut: z.number().int().nonnegative(),
|
|
64
|
+
contentHash: z.string().regex(/^[a-f0-9]{64}$/),
|
|
65
|
+
});
|
|
66
|
+
const rememberedSchema = z.strictObject({
|
|
67
|
+
instanceId: z.string(), streamId: z.string(), sequence: z.number().int().positive(), fingerprint: z.string(),
|
|
68
|
+
preparedAtMs: z.number().int().optional(), consentVersion: z.number().int().optional(),
|
|
69
|
+
});
|
|
70
|
+
export const runtimeStream = (instanceId, sourceDomain) => `cloud-runtime:${instanceId}:${sourceDomain}`;
|
|
71
|
+
function paused(home) {
|
|
72
|
+
const path = join(cloudHome(home), "control.json");
|
|
73
|
+
return existsSync(path) && z.object({ paused: z.boolean() })
|
|
74
|
+
.parse(JSON.parse(readFileSync(path, "utf8"))).paused;
|
|
75
|
+
}
|
|
76
|
+
function authorize(consent, commit) {
|
|
77
|
+
if (consent.mode !== "managed-content" || consent.scope.rootIds.length !== 0 ||
|
|
78
|
+
commit.projectId !== `device:${commit.deviceId}` || commit.rootId !== null ||
|
|
79
|
+
!contentKinds.every(kind => consent.contentKinds.includes(kind))) {
|
|
80
|
+
throw new CloudSyncError("runtime_consent_required");
|
|
81
|
+
}
|
|
82
|
+
authorizeUpload(consent, {
|
|
83
|
+
deviceId: commit.deviceId, projectId: commit.projectId, rootId: null,
|
|
84
|
+
consentVersion: commit.consentVersion, fromMs: commit.range.fromMs, toMs: commit.range.toMs,
|
|
85
|
+
backfill: commit.backfill, contentKinds,
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
/** Called only after CDC is caught up. Capture is requested only after full consent. */
|
|
89
|
+
export async function runCloudRuntimeStep(home, sourcePath, progress, capture, options = {}) {
|
|
90
|
+
if (progress.phase !== "caught-up" || progress.sourceDomain === undefined ||
|
|
91
|
+
progress.acknowledgedCut === undefined)
|
|
92
|
+
return progress;
|
|
93
|
+
const connection = readConnection(home);
|
|
94
|
+
if (connection === null || connection.instanceId !== progress.instanceId)
|
|
95
|
+
return progress;
|
|
96
|
+
const save = (fields) => {
|
|
97
|
+
const updated = { ...progress, updatedAtMs: Date.now(), ...fields };
|
|
98
|
+
atomicCloudJson(home, "status.json", updated);
|
|
99
|
+
return updated;
|
|
100
|
+
};
|
|
101
|
+
let blobs;
|
|
102
|
+
try {
|
|
103
|
+
if (paused(home))
|
|
104
|
+
return save({ phase: "paused" });
|
|
105
|
+
const self = await currentCloudDevice(connection, options.fetcher);
|
|
106
|
+
if (!self.ingestEnabled || !self.scopes.includes("activity:read") ||
|
|
107
|
+
!self.scopes.includes("activity:ingest") || self.consent.mode !== "managed-content" ||
|
|
108
|
+
!contentKinds.every(kind => self.consent.mode === "managed-content" && self.consent.contentKinds.includes(kind))) {
|
|
109
|
+
return progress;
|
|
110
|
+
}
|
|
111
|
+
const scope = cloudScope(connection, self.consent, false);
|
|
112
|
+
authorizeUpload(self.consent, { ...scope, contentKinds });
|
|
113
|
+
blobs = new CloudBlobs(join(cloudHome(home), "spool.sqlite"));
|
|
114
|
+
if (!blobs.acquire())
|
|
115
|
+
return save({ phase: "busy" });
|
|
116
|
+
const streamId = runtimeStream(connection.instanceId, progress.sourceDomain);
|
|
117
|
+
let batch = blobs.pending(streamId);
|
|
118
|
+
if (batch === null) {
|
|
119
|
+
const captured = await capture({
|
|
120
|
+
kind: "capture-runtime", id: randomUUID(), sourceDomain: progress.sourceDomain,
|
|
121
|
+
sourceCut: progress.acknowledgedCut,
|
|
122
|
+
});
|
|
123
|
+
if (captured === null)
|
|
124
|
+
return progress;
|
|
125
|
+
if (captured.sourceDomain !== progress.sourceDomain ||
|
|
126
|
+
captured.sourceCut !== progress.acknowledgedCut)
|
|
127
|
+
return save({ phase: "queued" });
|
|
128
|
+
const source = openDb(sourcePath);
|
|
129
|
+
try {
|
|
130
|
+
const head = journalHead(source);
|
|
131
|
+
if (head.sourceDomain !== captured.sourceDomain || head.cut !== captured.sourceCut) {
|
|
132
|
+
return save({ phase: "queued" });
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
finally {
|
|
136
|
+
source.close();
|
|
137
|
+
}
|
|
138
|
+
if (paused(home))
|
|
139
|
+
return save({ phase: "paused" });
|
|
140
|
+
// Serialization occurs only in this worker, after the explicit full grant.
|
|
141
|
+
const context = analysisRuntimeSchema.parse(captured.context);
|
|
142
|
+
const fingerprint = sha256(JSON.stringify({
|
|
143
|
+
sourceDomain: captured.sourceDomain, sourceCut: captured.sourceCut,
|
|
144
|
+
activeRequests: [...context.activeRequests].sort(), ingest: context.ingest, tz: context.tz,
|
|
145
|
+
}));
|
|
146
|
+
const statePath = join(cloudHome(home), "runtime-state.json");
|
|
147
|
+
const remembered = existsSync(statePath)
|
|
148
|
+
? rememberedSchema.parse(JSON.parse(readFileSync(statePath, "utf8"))) : null;
|
|
149
|
+
if (remembered?.instanceId === connection.instanceId && remembered.streamId === streamId &&
|
|
150
|
+
remembered.sequence === blobs.cursor(streamId)?.ack_sequence &&
|
|
151
|
+
remembered.fingerprint === fingerprint && remembered.consentVersion === scope.consentVersion &&
|
|
152
|
+
remembered.preparedAtMs !== undefined &&
|
|
153
|
+
(options.now?.() ?? Date.now()) - remembered.preparedAtMs <
|
|
154
|
+
Math.min(self.consent.retention.contentDays, self.consent.retention.projectionDays) *
|
|
155
|
+
86_400_000 / 2)
|
|
156
|
+
return progress;
|
|
157
|
+
// Start the renewal clock before upload, not at ACK: a lost receipt may
|
|
158
|
+
// leave an exact pending request retrying long after remote acceptance.
|
|
159
|
+
const preparedAtMs = options.now?.() ?? Date.now();
|
|
160
|
+
const bytes = Buffer.from(JSON.stringify(context));
|
|
161
|
+
const common = {
|
|
162
|
+
schemaVersion: 2, batchId: randomUUID(), deviceId: connection.deviceId,
|
|
163
|
+
sourceDomain: captured.sourceDomain, generation: connection.generation,
|
|
164
|
+
consentVersion: scope.consentVersion, projectId: scope.projectId, rootId: scope.rootId,
|
|
165
|
+
range: { fromMs: scope.fromMs, toMs: scope.toMs }, backfill: scope.backfill,
|
|
166
|
+
};
|
|
167
|
+
const refs = [];
|
|
168
|
+
const chunks = [];
|
|
169
|
+
for (let offset = 0; offset < bytes.length; offset += MAX_CONTENT_CHUNK_BYTES) {
|
|
170
|
+
const part = bytes.subarray(offset, offset + MAX_CONTENT_CHUNK_BYTES);
|
|
171
|
+
const chunk = parseContentChunkUpload({
|
|
172
|
+
...common, index: chunks.length, revision: captured.sourceCut,
|
|
173
|
+
plaintextHash: sha256(part), bytes: part.length, body: part.toString("base64"),
|
|
174
|
+
});
|
|
175
|
+
refs.push(blobs.put(Buffer.from(JSON.stringify(chunk))));
|
|
176
|
+
chunks.push({ index: chunk.index, revision: chunk.revision,
|
|
177
|
+
plaintextHash: chunk.plaintextHash, bytes: chunk.bytes });
|
|
178
|
+
}
|
|
179
|
+
// P6's transaction publishes the exact request and pins every chunk blob
|
|
180
|
+
// together. An interrupted preparatory put is an unreferenced local orphan.
|
|
181
|
+
batch = blobs.enqueue({
|
|
182
|
+
streamId, batchId: common.batchId, sourceDomain: captured.sourceDomain,
|
|
183
|
+
consent: self.consent, scope: { ...scope, contentKinds },
|
|
184
|
+
journal: { from: blobs.cursor(streamId)?.ack_cut ?? 0, through: captured.sourceCut },
|
|
185
|
+
blobHashes: refs.map(ref => ref.hash),
|
|
186
|
+
build: sequence => {
|
|
187
|
+
const commit = runtimeCommitSchema.parse({
|
|
188
|
+
...common, schemaVersion: 1, sequence, sourceCut: captured.sourceCut,
|
|
189
|
+
capturedAtMs: context.capturedAtMs, chunks, totalBytes: bytes.length, contentHash: sha256(bytes),
|
|
190
|
+
});
|
|
191
|
+
authorize(self.consent, commit);
|
|
192
|
+
return { kind: "runtime", body: JSON.stringify(commit), refs, fingerprint, preparedAtMs };
|
|
193
|
+
},
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
if (batch === null)
|
|
197
|
+
return progress;
|
|
198
|
+
const job = jobSchema.parse(JSON.parse(batch.payload));
|
|
199
|
+
const commit = runtimeCommitSchema.parse(JSON.parse(job.body));
|
|
200
|
+
if (commit.deviceId !== connection.deviceId || commit.generation !== connection.generation ||
|
|
201
|
+
commit.sourceDomain !== progress.sourceDomain || commit.batchId !== batch.batch_id ||
|
|
202
|
+
commit.sequence !== batch.sequence || commit.sourceCut !== batch.journal_through ||
|
|
203
|
+
job.refs.length !== commit.chunks.length)
|
|
204
|
+
throw new CloudSyncError("invalid_spool_job");
|
|
205
|
+
authorize(self.consent, commit);
|
|
206
|
+
const http = new CloudHttp(connection.origin, connection.token, options.fetcher);
|
|
207
|
+
for (const [index, ref] of job.refs.entries()) {
|
|
208
|
+
if (paused(home))
|
|
209
|
+
return save({ phase: "paused" });
|
|
210
|
+
const bytes = blobs.get(ref.hash);
|
|
211
|
+
if (bytes.length !== ref.bytes)
|
|
212
|
+
throw new CloudSyncError("spool_blob_corrupt");
|
|
213
|
+
const body = new TextDecoder("utf-8", { fatal: true }).decode(bytes);
|
|
214
|
+
const chunk = parseContentChunkUpload(JSON.parse(body));
|
|
215
|
+
const expected = commit.chunks[index];
|
|
216
|
+
if (chunk.batchId !== commit.batchId || chunk.sourceDomain !== commit.sourceDomain ||
|
|
217
|
+
chunk.deviceId !== commit.deviceId || chunk.generation !== commit.generation ||
|
|
218
|
+
expected?.plaintextHash !== chunk.plaintextHash || expected.bytes !== chunk.bytes ||
|
|
219
|
+
expected.revision !== chunk.revision || expected.index !== chunk.index) {
|
|
220
|
+
throw new CloudSyncError("spool_blob_corrupt");
|
|
221
|
+
}
|
|
222
|
+
blobs.renew();
|
|
223
|
+
let value;
|
|
224
|
+
try {
|
|
225
|
+
value = await http.request("/v1/activity/chunks", { body });
|
|
226
|
+
}
|
|
227
|
+
catch (error) {
|
|
228
|
+
// Retention may have retired an already accepted request after ACK loss.
|
|
229
|
+
// Only the exact commit receipt below can acknowledge it; an uncommitted
|
|
230
|
+
// purged batch still fails finalization and keeps its local pending job.
|
|
231
|
+
if (error instanceof CloudRequestError && error.status === 409 && error.code === "content_expired")
|
|
232
|
+
break;
|
|
233
|
+
// Consent moved on the server after this step read it: re-read the
|
|
234
|
+
// device consent and retry this chunk once under the current version.
|
|
235
|
+
if (error instanceof CloudRequestError && error.status === 409 && error.code === "consent_conflict") {
|
|
236
|
+
authorize((await currentCloudDevice(connection, options.fetcher)).consent, commit);
|
|
237
|
+
value = await http.request("/v1/activity/chunks", { body });
|
|
238
|
+
}
|
|
239
|
+
else
|
|
240
|
+
throw error;
|
|
241
|
+
}
|
|
242
|
+
const ack = z.strictObject({ stored: z.literal(true), chunk: contentChunkRefSchema }).parse(value);
|
|
243
|
+
blobs.renew();
|
|
244
|
+
if (JSON.stringify(ack.chunk) !== JSON.stringify(expected))
|
|
245
|
+
throw new CloudSyncError("invalid_remote_receipt");
|
|
246
|
+
}
|
|
247
|
+
if (paused(home))
|
|
248
|
+
return save({ phase: "paused" });
|
|
249
|
+
blobs.renew();
|
|
250
|
+
let receiptValue;
|
|
251
|
+
try {
|
|
252
|
+
receiptValue = await http.request("/v1/activity/context", { body: job.body });
|
|
253
|
+
}
|
|
254
|
+
catch (error) {
|
|
255
|
+
// Same consent re-read as the chunk upload above: one retry under the
|
|
256
|
+
// current server-side consent version.
|
|
257
|
+
if (error instanceof CloudRequestError && error.status === 409 && error.code === "consent_conflict") {
|
|
258
|
+
authorize((await currentCloudDevice(connection, options.fetcher)).consent, commit);
|
|
259
|
+
receiptValue = await http.request("/v1/activity/context", { body: job.body });
|
|
260
|
+
}
|
|
261
|
+
else
|
|
262
|
+
throw error;
|
|
263
|
+
}
|
|
264
|
+
const receipt = receiptSchema.parse(receiptValue);
|
|
265
|
+
blobs.renew();
|
|
266
|
+
if (receipt.sequence !== commit.sequence || receipt.sourceCut !== commit.sourceCut ||
|
|
267
|
+
receipt.contentHash !== commit.contentHash)
|
|
268
|
+
throw new CloudSyncError("invalid_remote_receipt");
|
|
269
|
+
blobs.acknowledge({ streamId, sequence: batch.sequence, payloadHash: batch.payload_hash });
|
|
270
|
+
atomicCloudJson(home, "runtime-state.json", {
|
|
271
|
+
instanceId: connection.instanceId, streamId, sequence: receipt.sequence, fingerprint: job.fingerprint,
|
|
272
|
+
preparedAtMs: job.preparedAtMs ?? 0, consentVersion: commit.consentVersion,
|
|
273
|
+
});
|
|
274
|
+
blobs.prune();
|
|
275
|
+
return progress;
|
|
276
|
+
}
|
|
277
|
+
catch (error) {
|
|
278
|
+
if (error instanceof CloudRequestError)
|
|
279
|
+
return save({
|
|
280
|
+
phase: error.state === "quarantined" ? "quarantine" : error.state,
|
|
281
|
+
code: error.code, retryAtMs: error.retryAtMs ?? undefined, attempts: progress.attempts + 1,
|
|
282
|
+
});
|
|
283
|
+
if (error instanceof z.ZodError)
|
|
284
|
+
return save({ phase: "quarantine", code: "invalid_contract" });
|
|
285
|
+
if (error instanceof CloudSyncError || error instanceof SyncError || error instanceof ActivityContractError) {
|
|
286
|
+
return save({ phase: /consent/.test(error.code) ? "consent-required" : "quarantine", code: error.code });
|
|
287
|
+
}
|
|
288
|
+
throw error;
|
|
289
|
+
}
|
|
290
|
+
finally {
|
|
291
|
+
blobs?.close();
|
|
292
|
+
}
|
|
293
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
// P4's immutable outbox/ACK schema is reused in this separate physical spool.
|
|
2
|
+
// Foreign-key references make pending request blobs and cursor commits atomic.
|
|
3
|
+
export const CLOUD_SCHEMA = `
|
|
4
|
+
CREATE TABLE IF NOT EXISTS blobs(
|
|
5
|
+
hash TEXT PRIMARY KEY,bytes BLOB NOT NULL,nbytes INTEGER NOT NULL CHECK(nbytes>=0)
|
|
6
|
+
);
|
|
7
|
+
CREATE TABLE IF NOT EXISTS lease(
|
|
8
|
+
id INTEGER PRIMARY KEY CHECK(id=1),owner TEXT NOT NULL,epoch INTEGER NOT NULL,until_ms INTEGER NOT NULL
|
|
9
|
+
);
|
|
10
|
+
INSERT OR IGNORE INTO lease VALUES(1,'',0,0);
|
|
11
|
+
CREATE TABLE IF NOT EXISTS sync_streams(
|
|
12
|
+
stream_id TEXT PRIMARY KEY,source_domain TEXT NOT NULL,device_id TEXT NOT NULL,
|
|
13
|
+
next_sequence INTEGER NOT NULL DEFAULT 1 CHECK(next_sequence BETWEEN 1 AND 9007199254740991),
|
|
14
|
+
ack_sequence INTEGER NOT NULL DEFAULT 0,ack_hash TEXT,ack_cut INTEGER NOT NULL DEFAULT 0,
|
|
15
|
+
CHECK(ack_sequence<next_sequence)
|
|
16
|
+
);
|
|
17
|
+
CREATE TABLE IF NOT EXISTS sync_outbox(
|
|
18
|
+
stream_id TEXT NOT NULL REFERENCES sync_streams(stream_id),sequence INTEGER NOT NULL,
|
|
19
|
+
batch_id TEXT NOT NULL UNIQUE,payload TEXT NOT NULL,payload_hash TEXT NOT NULL,
|
|
20
|
+
payload_bytes INTEGER NOT NULL CHECK(payload_bytes BETWEEN 1 AND 262144),
|
|
21
|
+
journal_from INTEGER NOT NULL,journal_through INTEGER NOT NULL CHECK(journal_through>=journal_from),
|
|
22
|
+
PRIMARY KEY(stream_id,sequence)
|
|
23
|
+
);
|
|
24
|
+
CREATE TRIGGER IF NOT EXISTS sync_outbox_immutable BEFORE UPDATE ON sync_outbox BEGIN
|
|
25
|
+
SELECT RAISE(ABORT,'sync outbox is immutable');
|
|
26
|
+
END;
|
|
27
|
+
CREATE TABLE IF NOT EXISTS sync_outbox_blobs(
|
|
28
|
+
stream_id TEXT NOT NULL,sequence INTEGER NOT NULL,hash TEXT NOT NULL REFERENCES blobs(hash),
|
|
29
|
+
PRIMARY KEY(stream_id,sequence,hash),
|
|
30
|
+
FOREIGN KEY(stream_id,sequence) REFERENCES sync_outbox(stream_id,sequence) ON DELETE CASCADE
|
|
31
|
+
);
|
|
32
|
+
`;
|