skillar 0.2.1 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/NOTICE +280 -0
  2. package/README.md +37 -5
  3. package/dist/analysis/finding.js +8 -190
  4. package/dist/analysis/kernel.js +896 -0
  5. package/dist/analysis/tree.js +7 -154
  6. package/dist/analysis/usage.js +9 -394
  7. package/dist/api/flows.js +42 -212
  8. package/dist/api/report-kernel.js +1063 -0
  9. package/dist/api/routes.js +12 -2
  10. package/dist/catalog/fields.js +138 -146
  11. package/dist/catalog/watcher.js +7 -2
  12. package/dist/cli/init.js +45 -0
  13. package/dist/cli/main.js +76 -3
  14. package/dist/cli/private-skills.js +122 -0
  15. package/dist/cli/serve.js +15 -1
  16. package/dist/cli/sync.js +124 -0
  17. package/dist/dashboard/NOTICE +280 -0
  18. package/dist/dashboard/assets/app.js +737 -275
  19. package/dist/dashboard/assets/astryx-reset.css +407 -0
  20. package/dist/dashboard/assets/astryx.css +1077 -0
  21. package/dist/dashboard/assets/charts.css +54 -0
  22. package/dist/dashboard/assets/chunk-TPGDA2J4.js +1 -0
  23. package/dist/dashboard/assets/chunk-YLWHVDMP.js +36 -0
  24. package/dist/dashboard/assets/controls.css +207 -0
  25. package/dist/dashboard/assets/overlays.css +132 -0
  26. package/dist/dashboard/assets/p1.css +135 -0
  27. package/dist/dashboard/assets/shell.css +71 -0
  28. package/dist/dashboard/assets/skillar-theme.css +692 -0
  29. package/dist/dashboard/assets/styles.css +224 -121
  30. package/dist/dashboard/assets/tokens.css +278 -0
  31. package/dist/dashboard/assets/widgets.css +83 -0
  32. package/dist/dashboard/index.html +11 -2
  33. package/dist/dashboard/viewers/viewers.css +61 -0
  34. package/dist/ingest/attribution.js +5 -1
  35. package/dist/ingest/pipeline.js +2 -0
  36. package/dist/ingest/request-recorder.js +39 -0
  37. package/dist/mcp/dispatch.js +2 -2
  38. package/dist/mcp/http.js +43 -3
  39. package/dist/mcp/skills-extension.js +42 -15
  40. package/dist/mcp/stdio-proxy.js +24 -3
  41. package/dist/mcp/tools.js +38 -24
  42. package/dist/packs/claude-code.js +22 -11
  43. package/dist/packs/cursor.js +9 -4
  44. package/dist/packs/gemini.js +9 -4
  45. package/dist/packs/json-config.js +41 -0
  46. package/dist/packs/omo.js +15 -6
  47. package/dist/request-status.js +81 -0
  48. package/dist/search/fts.js +93 -58
  49. package/dist/search/fusion.js +49 -47
  50. package/dist/search/guidance.js +18 -6
  51. package/dist/search/index.js +291 -70
  52. package/dist/search/lexical.js +71 -63
  53. package/dist/server.js +12 -0
  54. package/dist/skills/apply.js +130 -0
  55. package/dist/skills/capture.js +54 -0
  56. package/dist/skills/cloud-plan.js +57 -0
  57. package/dist/skills/cloud.js +81 -0
  58. package/dist/skills/fs.js +133 -0
  59. package/dist/skills/package.js +734 -0
  60. package/dist/skills/plan.js +83 -0
  61. package/dist/skills/receipt.js +44 -0
  62. package/dist/skills/rollback.js +119 -0
  63. package/dist/skills/transfer.js +51 -0
  64. package/dist/store/db.js +7 -1
  65. package/dist/store/execution.js +2 -34
  66. package/dist/store/export.js +25 -4
  67. package/dist/store/migrations/007_sync.sql +49 -0
  68. package/dist/store/migrations/008_connector_commands.sql +12 -0
  69. package/dist/store/queries.js +33 -7
  70. package/dist/store/writes.js +38 -5
  71. package/dist/sync/cloud-blobs.js +175 -0
  72. package/dist/sync/cloud-commands.js +101 -0
  73. package/dist/sync/cloud-connection.js +105 -0
  74. package/dist/sync/cloud-delivery.js +114 -0
  75. package/dist/sync/cloud-documents.js +103 -0
  76. package/dist/sync/cloud-export.js +67 -0
  77. package/dist/sync/cloud-http.js +150 -0
  78. package/dist/sync/cloud-jobs.js +94 -0
  79. package/dist/sync/cloud-pending.js +143 -0
  80. package/dist/sync/cloud-prepare.js +101 -0
  81. package/dist/sync/cloud-restore.js +136 -0
  82. package/dist/sync/cloud-rollback.js +24 -0
  83. package/dist/sync/cloud-runner.js +86 -0
  84. package/dist/sync/cloud-runtime.js +293 -0
  85. package/dist/sync/cloud-schema.js +32 -0
  86. package/dist/sync/cloud-stage.js +112 -0
  87. package/dist/sync/cloud-wire.js +693 -0
  88. package/dist/sync/cloud-worker.js +57 -0
  89. package/dist/sync/cloud.js +157 -0
  90. package/dist/sync/command-apply.js +140 -0
  91. package/dist/sync/command-control.js +25 -0
  92. package/dist/sync/command-wire.js +437 -0
  93. package/dist/sync/consent.js +130 -0
  94. package/dist/sync/journal.js +43 -0
  95. package/dist/sync/outbox.js +134 -0
  96. package/dist/sync/restore-rows.js +125 -0
  97. package/dist/sync/restore.js +174 -0
  98. package/dist/sync/snapshot.js +78 -0
  99. package/dist/sync/spool-request.js +103 -0
  100. package/dist/sync/tables.js +27 -0
  101. package/dist/sync/triggers.js +34 -0
  102. package/dist/workflow/compare.js +3 -43
  103. package/docs/cloud-sync.md +94 -0
  104. package/docs/private-skills.md +75 -0
  105. package/docs/sync-p4.md +102 -0
  106. package/docs/sync-restore.md +167 -0
  107. package/package.json +18 -5
  108. package/src/store/migrations/007_sync.sql +49 -0
  109. package/src/store/migrations/008_connector_commands.sql +12 -0
  110. package/dist/dashboard/assets/chunk-5DFWNJC2.js +0 -1
  111. package/dist/dashboard/assets/chunk-S6B2HJQA.js +0 -17
@@ -0,0 +1,134 @@
1
+ import { createHash, randomUUID } from "node:crypto";
2
+ import { z } from "zod";
3
+ import { authorizeUpload, consentSchema, transformMutation, uploadScopeSchema, } from "./consent.js";
4
+ import { journalHead } from "./journal.js";
5
+ import { chunkLimits } from "./snapshot.js";
6
+ import { mutationSchema, SyncError } from "./tables.js";
7
+ const counter = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER);
8
+ const streamSchema = z.object({
9
+ source_domain: z.string(), device_id: z.string(), next_sequence: counter, ack_sequence: counter,
10
+ ack_hash: z.string().nullable(), ack_cut: counter,
11
+ });
12
+ const storedBatch = z.object({
13
+ stream_id: z.string(), sequence: counter, batch_id: z.string(),
14
+ payload: z.string(), payload_hash: z.string(), payload_bytes: counter,
15
+ journal_from: counter, journal_through: counter,
16
+ });
17
+ /**
18
+ * No implicit capture-level consent and no background/network work. The caller
19
+ * supplies a bounded page and proven source membership. Only transformed bytes
20
+ * enter the outbox. Any failure rolls back sequence allocation as well.
21
+ */
22
+ export function spoolBatch(db, request) {
23
+ const consent = consentSchema.parse(request.consent ?? { mode: "local-only" });
24
+ if (consent.mode === "local-only")
25
+ return null;
26
+ const scope = uploadScopeSchema.parse(request.scope);
27
+ authorizeUpload(consent, scope);
28
+ const limits = chunkLimits(request.limits);
29
+ const records = z.array(mutationSchema).min(1).max(limits.maxRecords).parse(request.records);
30
+ const journal = z.strictObject({ from: counter, through: counter })
31
+ .refine(value => value.through >= value.from).parse(request.journal);
32
+ const streamId = z.string().min(1).max(512).parse(request.streamId);
33
+ const transformed = records.map(record => {
34
+ if (record.operation === "delete" && record.beforeValues === null) {
35
+ throw new SyncError("scope_required");
36
+ }
37
+ const membership = request.scopeFor(record);
38
+ if (membership.projectId !== scope.projectId || membership.rootId !== scope.rootId) {
39
+ throw new SyncError("scope_required");
40
+ }
41
+ if (record.revision < journal.from || record.revision > journal.through) {
42
+ throw new SyncError("invalid_cut");
43
+ }
44
+ return transformMutation(record, scope);
45
+ });
46
+ const quota = z.strictObject({
47
+ maxBatches: z.number().int().positive().default(1024),
48
+ maxBytes: z.number().int().positive().default(64 * 1024 * 1024),
49
+ }).parse(request.quota ?? {});
50
+ const head = journalHead(db);
51
+ if (journal.through > head.cut)
52
+ throw new SyncError("invalid_cut");
53
+ db.exec("BEGIN IMMEDIATE");
54
+ try {
55
+ db.prepare(`INSERT INTO sync_streams(stream_id,source_domain,device_id) VALUES (?,?,?)
56
+ ON CONFLICT(stream_id) DO NOTHING`).run(streamId, head.sourceDomain, scope.deviceId);
57
+ const stream = streamSchema.parse(db.prepare("SELECT * FROM sync_streams WHERE stream_id=?").get(streamId));
58
+ if (stream.source_domain !== head.sourceDomain || stream.device_id !== scope.deviceId) {
59
+ throw new SyncError("invalid_stream");
60
+ }
61
+ const tail = z.object({ through: counter }).parse(db.prepare(`SELECT
62
+ COALESCE((SELECT journal_through FROM sync_outbox WHERE stream_id=?
63
+ ORDER BY sequence DESC LIMIT 1), ?) AS through`).get(streamId, stream.ack_cut));
64
+ if (journal.from !== tail.through)
65
+ throw new SyncError("invalid_cut");
66
+ const batchId = randomUUID();
67
+ const payload = JSON.stringify({
68
+ format: "skillar-local-outbox-v1", batchId, sourceDomain: head.sourceDomain,
69
+ sequence: stream.next_sequence, scope, journal, records: transformed,
70
+ });
71
+ const bytes = Buffer.byteLength(payload);
72
+ if (bytes > limits.maxBytes)
73
+ throw new SyncError("record_too_large");
74
+ const usage = z.object({ batches: counter, bytes: counter }).parse(db.prepare("SELECT COUNT(*) AS batches, COALESCE(SUM(payload_bytes),0) AS bytes FROM sync_outbox").get());
75
+ if (usage.batches >= quota.maxBatches || usage.bytes + bytes > quota.maxBytes) {
76
+ throw new SyncError("outbox_full");
77
+ }
78
+ const batch = {
79
+ stream_id: streamId, sequence: stream.next_sequence, batch_id: batchId,
80
+ payload, payload_hash: createHash("sha256").update(payload).digest("hex"),
81
+ payload_bytes: bytes, journal_from: journal.from, journal_through: journal.through,
82
+ };
83
+ db.prepare(`INSERT INTO sync_outbox VALUES (?,?,?,?,?,?,?,?)`).run(batch.stream_id, batch.sequence, batch.batch_id, batch.payload, batch.payload_hash, batch.payload_bytes, batch.journal_from, batch.journal_through);
84
+ db.prepare("UPDATE sync_streams SET next_sequence=next_sequence+1 WHERE stream_id=?").run(streamId);
85
+ db.exec("COMMIT");
86
+ return batch;
87
+ }
88
+ catch (error) {
89
+ db.exec("ROLLBACK");
90
+ throw error;
91
+ }
92
+ }
93
+ /** Only the oldest unacknowledged batch is eligible for sending. */
94
+ export function pendingBatch(db, streamId) {
95
+ if (db.isTransaction)
96
+ throw new SyncError("transaction_active");
97
+ const row = db.prepare(`SELECT o.* FROM sync_outbox o JOIN sync_streams s USING(stream_id)
98
+ WHERE o.stream_id=? AND o.sequence=s.ack_sequence+1`).get(streamId);
99
+ return row === undefined ? null : storedBatch.parse(row);
100
+ }
101
+ /** ACK means an externally verified durable receipt; there is no sender in P4. */
102
+ export function acknowledgeBatch(db, receipt, assertLease) {
103
+ if (db.isTransaction)
104
+ throw new SyncError("transaction_active");
105
+ db.exec("BEGIN IMMEDIATE");
106
+ try {
107
+ assertLease?.();
108
+ const raw = db.prepare("SELECT * FROM sync_streams WHERE stream_id=?").get(receipt.streamId);
109
+ if (raw === undefined)
110
+ throw new SyncError("ack_out_of_order");
111
+ const stream = streamSchema.parse(raw);
112
+ if (receipt.sequence === stream.ack_sequence && receipt.payloadHash === stream.ack_hash) {
113
+ db.exec("COMMIT");
114
+ return;
115
+ }
116
+ const rawBatch = db.prepare("SELECT * FROM sync_outbox WHERE stream_id=? AND sequence=?")
117
+ .get(receipt.streamId, receipt.sequence);
118
+ if (receipt.sequence !== stream.ack_sequence + 1 || rawBatch === undefined) {
119
+ throw new SyncError("ack_out_of_order");
120
+ }
121
+ const batch = storedBatch.parse(rawBatch);
122
+ if (batch.payload_hash !== receipt.payloadHash)
123
+ throw new SyncError("ack_out_of_order");
124
+ db.prepare(`UPDATE sync_streams SET ack_sequence=?,ack_hash=?,ack_cut=? WHERE stream_id=?`)
125
+ .run(receipt.sequence, receipt.payloadHash, batch.journal_through, receipt.streamId);
126
+ db.prepare("DELETE FROM sync_outbox WHERE stream_id=? AND sequence=?")
127
+ .run(receipt.streamId, receipt.sequence);
128
+ db.exec("COMMIT");
129
+ }
130
+ catch (error) {
131
+ db.exec("ROLLBACK");
132
+ throw error;
133
+ }
134
+ }
@@ -0,0 +1,125 @@
1
+ import { createHash } from "node:crypto";
2
+ import { z } from "zod";
3
+ import { mutationSchema, scalarSchema, targetTables } from "./tables.js";
4
+ export class RestoreError extends Error {
5
+ code;
6
+ constructor(code) {
7
+ super(code);
8
+ this.code = code;
9
+ this.name = "RestoreError";
10
+ }
11
+ }
12
+ export const cutSchema = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER);
13
+ export const descriptorSchema = z.strictObject({
14
+ sourceDomain: z.string().min(1),
15
+ cut: cutSchema,
16
+ counts: z.record(z.enum(targetTables), cutSchema).optional(),
17
+ hash: z.string().regex(/^[a-f0-9]{64}$/).optional(),
18
+ }).readonly();
19
+ export function shape(db, table) {
20
+ const columns = db.prepare(`PRAGMA table_info("${table}")`).all().map(row => z.object({ name: z.string(), pk: z.number() }).parse(row));
21
+ return { columns: columns.map(c => c.name),
22
+ keys: columns.filter(c => c.pk > 0).sort((a, b) => a.pk - b.pk).map(c => c.name) };
23
+ }
24
+ export const identity = (row) => JSON.stringify([row.table, row.key]);
25
+ /** Parse the complete typed SQL image, never a partial projection/defaulted import. */
26
+ export function parseRecord(db, input) {
27
+ const row = mutationSchema.parse(input);
28
+ const { columns, keys } = shape(db, row.table);
29
+ const key = z.array(scalarSchema).parse(JSON.parse(row.key));
30
+ if (key.length !== keys.length || key.some(v => v === null) ||
31
+ JSON.stringify(key) !== row.key ||
32
+ (row.table === "load_candidates" ? row.sourceOrder !== null :
33
+ row.sourceOrder === null || !Number.isSafeInteger(row.sourceOrder))) {
34
+ throw new RestoreError("invalid_record");
35
+ }
36
+ switch (row.operation) {
37
+ case "delete":
38
+ if (row.values !== null)
39
+ throw new RestoreError("invalid_record");
40
+ break;
41
+ case "upsert": {
42
+ const values = row.values;
43
+ if (values === null || Object.keys(values).length !== columns.length ||
44
+ columns.some(c => !Object.hasOwn(values, c)) ||
45
+ keys.some((k, i) => values[k] !== key[i]))
46
+ throw new RestoreError("invalid_record");
47
+ break;
48
+ }
49
+ default: {
50
+ const unreachable = row.operation;
51
+ throw unreachable;
52
+ }
53
+ }
54
+ return row;
55
+ }
56
+ export function readRows(db, cut, clock) {
57
+ const rows = [];
58
+ for (const table of targetTables) {
59
+ if (table === "recording_clock" && clock) {
60
+ rows.push({ ...clock, revision: cut });
61
+ continue;
62
+ }
63
+ const { columns, keys } = shape(db, table);
64
+ const order = table === "load_candidates" ? keys.map(c => `"${c}"`).join(",") : "rowid";
65
+ for (const raw of db.prepare(`SELECT *, ${table === "load_candidates" ? "NULL" : "rowid"}
66
+ AS restore_source_order FROM "${table}" ORDER BY ${order}`).all()) {
67
+ const values = Object.fromEntries(columns.map(c => [c, scalarSchema.parse(raw[c])]));
68
+ rows.push(mutationSchema.parse({ table, key: JSON.stringify(keys.map(k => values[k])),
69
+ revision: cut, operation: "upsert", sourceOrder: raw.restore_source_order, values }));
70
+ }
71
+ }
72
+ return rows;
73
+ }
74
+ export function summary(rows) {
75
+ const counts = Object.fromEntries(targetTables.map(t => [t, rows.filter(r => r.table === t).length]));
76
+ return { counts, hash: createHash("sha256").update(JSON.stringify(rows)).digest("hex") };
77
+ }
78
+ /** All trigger DDL and writes share the caller's transaction, including rollback. */
79
+ export function replaceRows(db, rows) {
80
+ const triggers = db.prepare("SELECT name,sql FROM sqlite_schema WHERE type='trigger' ORDER BY rowid")
81
+ .all().map(row => z.object({ name: z.string(), sql: z.string() }).parse(row));
82
+ for (const trigger of triggers)
83
+ db.exec(`DROP TRIGGER "${trigger.name.replaceAll('"', '""')}"`);
84
+ db.exec("PRAGMA defer_foreign_keys = ON");
85
+ // Delete children first so foreign-key cascades cannot remove restored images.
86
+ for (const table of [...targetTables].reverse()) {
87
+ if (table !== "recording_clock")
88
+ db.exec(`DELETE FROM "${table}"`);
89
+ }
90
+ for (const row of rows) {
91
+ if (row.table === "recording_clock")
92
+ continue;
93
+ const { columns } = shape(db, row.table);
94
+ const values = row.values;
95
+ if (values === null)
96
+ throw new RestoreError("invalid_record");
97
+ const withRowid = row.table !== "load_candidates";
98
+ const names = [...(withRowid ? ["rowid"] : []), ...columns];
99
+ db.prepare(`INSERT INTO "${row.table}" (${names.map(c => `"${c}"`).join(",")})
100
+ VALUES (${names.map(() => "?").join(",")})`)
101
+ .run(...(withRowid ? [row.sourceOrder] : []), ...columns.map(c => values[c] ?? null));
102
+ }
103
+ // FTS is derived state, not source history; do not copy source virtual-table internals.
104
+ db.exec(`DELETE FROM skills_fts;
105
+ INSERT INTO skills_fts(uri,name,description,when_to_use,examples,aliases_ko,body_head)
106
+ SELECT uri,name,description,when_to_use,examples_json,aliases_ko_json,body_head
107
+ FROM skills WHERE removed_at IS NULL`);
108
+ for (const trigger of triggers)
109
+ db.exec(trigger.sql);
110
+ const clock = rows.find(row => row.table === "recording_clock");
111
+ const expected = new Map(rows.map(row => [identity(row), row]));
112
+ const stored = readRows(db, 0, clock);
113
+ if (stored.length !== rows.length || stored.some(row => {
114
+ const original = expected.get(identity(row));
115
+ return !original || original.sourceOrder !== row.sourceOrder ||
116
+ Object.entries(row.values ?? {}).some(([name, value]) => original.values?.[name] !== value);
117
+ }))
118
+ throw new RestoreError("verification_failed");
119
+ }
120
+ export function checkDatabase(db) {
121
+ if (db.prepare("PRAGMA foreign_key_check").all().length > 0 ||
122
+ db.prepare("PRAGMA integrity_check").all().some(row => row.integrity_check !== "ok")) {
123
+ throw new RestoreError("verification_failed");
124
+ }
125
+ }
@@ -0,0 +1,174 @@
1
+ import { createHash } from "node:crypto";
2
+ import { z } from "zod";
3
+ import { mutationSchema, targetTables } from "./tables.js";
4
+ import { checkDatabase, cutSchema, descriptorSchema, identity, parseRecord, readRows, replaceRows, RestoreError, summary, } from "./restore-rows.js";
5
+ export { RestoreError };
6
+ const stateSchema = z.object({
7
+ source_domain: z.string(), cut: cutSchema, clock_json: z.string(),
8
+ });
9
+ const pageSchema = z.strictObject({
10
+ snapshotId: z.string().min(1), sourceDomain: z.string().min(1), cut: cutSchema,
11
+ index: cutSchema, records: z.array(mutationSchema), bytes: cutSchema,
12
+ hash: z.string().regex(/^[a-f0-9]{64}$/),
13
+ });
14
+ function state(db) {
15
+ if (!db.prepare("SELECT 1 FROM sqlite_schema WHERE name='sync_restore_state'").get()) {
16
+ throw new RestoreError("restore_missing");
17
+ }
18
+ return stateSchema.parse(db.prepare("SELECT * FROM sync_restore_state WHERE singleton=1").get());
19
+ }
20
+ function idle(db) {
21
+ if (db.isTransaction)
22
+ throw new RestoreError("destination_busy");
23
+ }
24
+ function pristine(db) {
25
+ if (["sync_journal", "sync_streams", "sync_outbox", "meta"].some(table => db.prepare(`SELECT 1 FROM "${table}" LIMIT 1`).get()) ||
26
+ db.prepare("SELECT 1 FROM sqlite_sequence WHERE name='sync_journal' AND seq>0").get() ||
27
+ db.prepare("SELECT 1 FROM recording_clock WHERE position<>0").get()) {
28
+ throw new RestoreError("destination_used");
29
+ }
30
+ }
31
+ function transaction(db, run) {
32
+ idle(db);
33
+ db.exec("BEGIN IMMEDIATE");
34
+ try {
35
+ const result = run();
36
+ db.exec("COMMIT");
37
+ return result;
38
+ }
39
+ catch (error) {
40
+ db.exec("ROLLBACK");
41
+ throw error;
42
+ }
43
+ }
44
+ function saveState(db, descriptor, rows) {
45
+ const clocks = rows.filter(r => r.table === "recording_clock");
46
+ const clock = clocks[0];
47
+ if (clocks.length !== 1 || !clock || clock.key !== "[1]" || clock.sourceOrder !== 1 ||
48
+ clock.values?.singleton !== 1 ||
49
+ !z.string().min(1).safeParse(clock.values.domain).success ||
50
+ !cutSchema.safeParse(clock.values.position).success)
51
+ throw new RestoreError("invalid_snapshot");
52
+ const local = z.object({ domain: z.string() }).parse(db.prepare("SELECT domain FROM recording_clock WHERE singleton=1").get());
53
+ const sync = z.object({ source_domain: z.string() }).parse(db.prepare("SELECT source_domain FROM sync_identity WHERE singleton=1").get());
54
+ if (descriptor.sourceDomain === sync.source_domain || clock.values.domain === local.domain) {
55
+ throw new RestoreError("destination_used");
56
+ }
57
+ db.exec(`CREATE TABLE IF NOT EXISTS sync_restore_state (
58
+ singleton INTEGER PRIMARY KEY CHECK(singleton=1), source_domain TEXT NOT NULL,
59
+ cut INTEGER NOT NULL, clock_json TEXT NOT NULL)`);
60
+ db.prepare(`INSERT INTO sync_restore_state VALUES(1,?,?,?)
61
+ ON CONFLICT(singleton) DO UPDATE SET source_domain=excluded.source_domain,
62
+ cut=excluded.cut,clock_json=excluded.clock_json`)
63
+ .run(descriptor.sourceDomain, descriptor.cut, JSON.stringify(clock));
64
+ }
65
+ function verifyRows(db, descriptor) {
66
+ const saved = state(db);
67
+ if (saved.source_domain !== descriptor.sourceDomain || saved.cut !== descriptor.cut) {
68
+ throw new RestoreError("verification_failed");
69
+ }
70
+ const rows = readRows(db, saved.cut, mutationSchema.parse(JSON.parse(saved.clock_json)));
71
+ const result = summary(rows);
72
+ if ((descriptor.hash !== undefined && descriptor.hash !== result.hash) ||
73
+ (descriptor.counts !== undefined &&
74
+ targetTables.some(table => descriptor.counts?.[table] !== result.counts[table]))) {
75
+ throw new RestoreError("verification_failed");
76
+ }
77
+ checkDatabase(db);
78
+ return { sourceDomain: saved.source_domain, cut: saved.cut, ...result };
79
+ }
80
+ /** Hash/count verification uses canonical logicalSnapshot order, substituting source clock provenance. */
81
+ export function verify(db, input) {
82
+ const descriptor = descriptorSchema.parse(input);
83
+ idle(db);
84
+ db.exec("BEGIN");
85
+ try {
86
+ return verifyRows(db, descriptor);
87
+ }
88
+ finally {
89
+ db.exec("ROLLBACK");
90
+ }
91
+ }
92
+ /** Fresh migrated database only. Iterator, parsing, SQL and verification failures roll back together. */
93
+ export function restoreSnapshot(db, input, pages) {
94
+ const descriptor = descriptorSchema.parse(input);
95
+ return transaction(db, () => {
96
+ pristine(db);
97
+ if (targetTables.some(table => table !== "recording_clock" &&
98
+ db.prepare(`SELECT 1 FROM "${table}" LIMIT 1`).get()) ||
99
+ db.prepare("SELECT 1 FROM skills_fts LIMIT 1").get() ||
100
+ db.prepare("SELECT 1 FROM sqlite_schema WHERE name='sync_restore_state'").get()) {
101
+ throw new RestoreError("destination_used");
102
+ }
103
+ const rows = new Map();
104
+ let index = 0;
105
+ let snapshotId;
106
+ for (const inputPage of pages) {
107
+ const page = pageSchema.parse(inputPage);
108
+ const payload = JSON.stringify(page.records);
109
+ snapshotId ??= page.snapshotId;
110
+ if (page.index !== index++ || page.snapshotId !== snapshotId ||
111
+ page.sourceDomain !== descriptor.sourceDomain || page.cut !== descriptor.cut ||
112
+ page.bytes !== Buffer.byteLength(payload) ||
113
+ page.hash !== createHash("sha256").update(payload).digest("hex")) {
114
+ throw new RestoreError("invalid_snapshot");
115
+ }
116
+ for (const inputRow of page.records) {
117
+ const row = parseRecord(db, inputRow);
118
+ const key = identity(row);
119
+ if (row.operation !== "upsert" || row.revision !== descriptor.cut || rows.has(key)) {
120
+ throw new RestoreError("invalid_snapshot");
121
+ }
122
+ rows.set(key, row);
123
+ }
124
+ }
125
+ if (index === 0)
126
+ throw new RestoreError("invalid_snapshot");
127
+ const records = [...rows.values()];
128
+ saveState(db, descriptor, records);
129
+ replaceRows(db, records);
130
+ return verifyRows(db, descriptor);
131
+ });
132
+ }
133
+ /** One complete committed source range; gather readJournal pages before calling. */
134
+ export function applyCatchup(db, input) {
135
+ const descriptor = descriptorSchema.parse({
136
+ sourceDomain: input.sourceDomain, cut: input.cut, counts: input.counts, hash: input.hash,
137
+ });
138
+ const after = cutSchema.parse(input.after);
139
+ return transaction(db, () => {
140
+ pristine(db);
141
+ const saved = state(db);
142
+ if (saved.source_domain !== descriptor.sourceDomain || saved.cut !== after ||
143
+ descriptor.cut < after)
144
+ throw new RestoreError("invalid_catchup");
145
+ const rows = new Map(readRows(db, descriptor.cut, mutationSchema.parse(JSON.parse(saved.clock_json))).map(row => [identity(row), row]));
146
+ let previous = after;
147
+ const seen = new Set();
148
+ for (const inputRow of input.records) {
149
+ const row = parseRecord(db, inputRow);
150
+ const key = identity(row);
151
+ if (row.revision <= previous || row.revision > descriptor.cut || seen.has(key)) {
152
+ throw new RestoreError("invalid_catchup");
153
+ }
154
+ previous = row.revision;
155
+ seen.add(key);
156
+ switch (row.operation) {
157
+ case "delete":
158
+ rows.delete(key);
159
+ break;
160
+ case "upsert":
161
+ rows.set(key, { ...row, revision: descriptor.cut });
162
+ break;
163
+ default: {
164
+ const unreachable = row.operation;
165
+ throw unreachable;
166
+ }
167
+ }
168
+ }
169
+ const records = [...rows.values()];
170
+ saveState(db, descriptor, records);
171
+ replaceRows(db, records);
172
+ return verifyRows(db, descriptor);
173
+ });
174
+ }
@@ -0,0 +1,78 @@
1
+ import { createHash, randomUUID } from "node:crypto";
2
+ import { z } from "zod";
3
+ import { mutationSchema, SyncError, targetTables } from "./tables.js";
4
+ export function chunkLimits(limits = {}) {
5
+ const parsed = z.strictObject({
6
+ maxRecords: z.number().int().min(1).max(100).default(100),
7
+ maxBytes: z.number().int().min(1).max(256 * 1024).default(256 * 1024),
8
+ }).safeParse(limits);
9
+ if (!parsed.success)
10
+ throw new SyncError("invalid_limits");
11
+ return parsed.data;
12
+ }
13
+ /**
14
+ * A dedicated reader is required. The generator owns its read transaction until
15
+ * exhaustion/return(), including pauses between chunks. Its first SELECT pins
16
+ * BOTH the journal cut and source rows to the same WAL snapshot.
17
+ */
18
+ export function* logicalSnapshot(db, options = {}) {
19
+ if (db.isTransaction)
20
+ throw new SyncError("transaction_active");
21
+ const limits = chunkLimits({
22
+ maxRecords: options.maxRecords, maxBytes: options.maxBytes,
23
+ });
24
+ const maxDuration = z.number().positive().finite().parse(options.maxDurationMs ?? 30_000);
25
+ const deadline = performance.now() + maxDuration;
26
+ const snapshotId = randomUUID();
27
+ db.exec("BEGIN");
28
+ try {
29
+ // journalHead cannot run inside transactions; this query is deliberately the
30
+ // first read, rather than using the pre-transaction head.
31
+ const pinned = z.object({ cut: z.number().int(), source_domain: z.string() }).parse(db.prepare(`SELECT source_domain, COALESCE((SELECT MAX(seq) FROM sync_journal),0) AS cut
32
+ FROM sync_identity`).get());
33
+ let index = 0;
34
+ let records = [];
35
+ const chunk = () => {
36
+ const payload = JSON.stringify(records);
37
+ return { snapshotId, sourceDomain: pinned.source_domain, cut: pinned.cut,
38
+ index: index++, records, bytes: Buffer.byteLength(payload),
39
+ hash: createHash("sha256").update(payload).digest("hex") };
40
+ };
41
+ for (const table of targetTables) {
42
+ const columns = db.prepare(`PRAGMA table_info("${table}")`).all().map(row => z.object({ name: z.string(), pk: z.number() }).parse(row));
43
+ const keys = columns.filter(c => c.pk > 0).sort((a, b) => a.pk - b.pk);
44
+ const withoutRowid = table === "load_candidates";
45
+ const key = `json_array(${keys.map(c => `"${c.name}"`).join(",")})`;
46
+ const values = `json_object(${columns.map(c => `'${c.name}',"${c.name}"`).join(",")})`;
47
+ const query = db.prepare(`SELECT ${key} AS entity_key, ${values} AS row_json,
48
+ ${withoutRowid ? "NULL" : "rowid"} AS source_order FROM "${table}"
49
+ ORDER BY ${withoutRowid ? keys.map(c => `"${c.name}"`).join(",") : "rowid"}`);
50
+ for (const raw of query.iterate()) {
51
+ if (performance.now() > deadline)
52
+ throw new SyncError("snapshot_expired");
53
+ const row = z.object({
54
+ entity_key: z.string(), row_json: z.string(), source_order: z.number().nullable(),
55
+ }).parse(raw);
56
+ const record = mutationSchema.parse({ table, key: row.entity_key, revision: pinned.cut,
57
+ operation: "upsert", sourceOrder: row.source_order, values: JSON.parse(row.row_json) });
58
+ if (Buffer.byteLength(JSON.stringify([record])) > limits.maxBytes) {
59
+ throw new SyncError("record_too_large");
60
+ }
61
+ if (records.length === limits.maxRecords ||
62
+ Buffer.byteLength(JSON.stringify([...records, record])) > limits.maxBytes) {
63
+ yield chunk();
64
+ records = [];
65
+ }
66
+ records.push(record);
67
+ }
68
+ }
69
+ if (records.length > 0)
70
+ yield chunk();
71
+ // Empty snapshots still communicate their verified cut and hash.
72
+ else if (index === 0)
73
+ yield chunk();
74
+ }
75
+ finally {
76
+ db.exec("ROLLBACK");
77
+ }
78
+ }
@@ -0,0 +1,103 @@
1
+ import { createHash } from "node:crypto";
2
+ import { z } from "zod";
3
+ import { authorizeUpload, consentSchema, uploadScopeSchema } from "./consent.js";
4
+ import { SyncError } from "./tables.js";
5
+ const counter = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER);
6
+ const streamSchema = z.object({
7
+ source_domain: z.string(), device_id: z.string(), next_sequence: counter,
8
+ ack_sequence: counter, ack_cut: counter,
9
+ });
10
+ const quotaSchema = z.object({
11
+ maxBatches: z.number().int().positive().default(1024),
12
+ maxBytes: z.number().int().positive().default(64 * 1024 * 1024),
13
+ });
14
+ /**
15
+ * Only a verified baseline/status receipt may initialize a new live cursor.
16
+ * Existing local pending bytes are never overwritten by a server status read.
17
+ */
18
+ export function initializeQueue(db, input) {
19
+ const sequence = counter.parse(input.sequence), cut = counter.parse(input.cut);
20
+ const sourceDomain = z.string().min(1).max(512).parse(input.sourceDomain);
21
+ const streamId = z.string().min(1).max(512).parse(input.streamId);
22
+ const deviceId = z.string().min(1).max(512).parse(input.deviceId);
23
+ db.prepare(`INSERT INTO sync_streams
24
+ (stream_id,source_domain,device_id,next_sequence,ack_sequence,ack_cut)
25
+ VALUES (?,?,?,?,?,?) ON CONFLICT(stream_id) DO NOTHING`)
26
+ .run(streamId, sourceDomain, deviceId, counter.parse(sequence + 1), sequence, cut);
27
+ const saved = streamSchema.parse(db.prepare("SELECT * FROM sync_streams WHERE stream_id=?").get(streamId));
28
+ if (saved.source_domain !== sourceDomain || saved.device_id !== deviceId ||
29
+ saved.ack_sequence !== sequence || saved.ack_cut !== cut) {
30
+ throw new SyncError("invalid_stream");
31
+ }
32
+ }
33
+ /**
34
+ * P6 reuses P4's immutable spool/ACK tables for already-consent-transformed
35
+ * request manifests. Raw blobs and references share this physical spool DB.
36
+ * build must be synchronous and perform no I/O; it receives the allocated seq.
37
+ */
38
+ export function spoolRequest(db, input) {
39
+ const consent = consentSchema.parse(input.consent);
40
+ if (consent.mode === "local-only")
41
+ return null;
42
+ if (consent.mode !== "managed-content")
43
+ throw new SyncError("content_not_allowed");
44
+ const scope = uploadScopeSchema.parse(input.scope);
45
+ authorizeUpload(consent, scope);
46
+ const streamId = z.string().min(1).max(512).parse(input.streamId);
47
+ const batchId = z.string().min(1).max(512).parse(input.batchId);
48
+ const journal = z.object({ from: counter, through: counter })
49
+ .refine(value => value.through >= value.from).parse(input.journal);
50
+ const quota = quotaSchema.parse(input.quota ?? {});
51
+ const sourceDomain = z.string().min(1).max(512).parse(input.sourceDomain);
52
+ db.exec("BEGIN IMMEDIATE");
53
+ try {
54
+ input.assertLease?.();
55
+ db.prepare(`INSERT INTO sync_streams(stream_id,source_domain,device_id)
56
+ VALUES (?,?,?) ON CONFLICT(stream_id) DO NOTHING`)
57
+ .run(streamId, sourceDomain, scope.deviceId);
58
+ const stream = streamSchema.parse(db.prepare("SELECT * FROM sync_streams WHERE stream_id=?").get(streamId));
59
+ if (stream.source_domain !== sourceDomain || stream.device_id !== scope.deviceId) {
60
+ throw new SyncError("invalid_stream");
61
+ }
62
+ const tail = z.object({ through: counter }).parse(db.prepare(`SELECT COALESCE(
63
+ (SELECT journal_through FROM sync_outbox WHERE stream_id=?
64
+ ORDER BY sequence DESC LIMIT 1),?) AS through`).get(streamId, stream.ack_cut));
65
+ if (journal.from !== tail.through)
66
+ throw new SyncError("invalid_cut");
67
+ const sequence = counter.parse(stream.next_sequence);
68
+ const payload = JSON.stringify(input.build(sequence));
69
+ if (payload === undefined)
70
+ throw new SyncError("invalid_stream");
71
+ const bytes = Buffer.byteLength(payload);
72
+ if (bytes > 256 * 1024)
73
+ throw new SyncError("record_too_large");
74
+ const used = z.object({ count: counter, bytes: counter }).parse(db.prepare(`SELECT
75
+ count(*) AS count,COALESCE(sum(payload_bytes),0) AS bytes FROM sync_outbox`).get());
76
+ const blobBytes = counter.parse(input.storedBlobBytes?.() ?? 0);
77
+ if (used.count >= quota.maxBatches || used.bytes + blobBytes + bytes > quota.maxBytes) {
78
+ throw new SyncError("outbox_full");
79
+ }
80
+ const batch = {
81
+ stream_id: streamId, sequence, batch_id: batchId, payload,
82
+ payload_hash: createHash("sha256").update(payload).digest("hex"), payload_bytes: bytes,
83
+ journal_from: journal.from, journal_through: journal.through,
84
+ };
85
+ db.prepare(`INSERT INTO sync_outbox
86
+ (stream_id,sequence,batch_id,payload,payload_hash,payload_bytes,journal_from,journal_through)
87
+ VALUES (?,?,?,?,?,?,?,?)`).run(streamId, sequence, batchId, payload, batch.payload_hash, bytes, journal.from, journal.through);
88
+ db.prepare("UPDATE sync_streams SET next_sequence=? WHERE stream_id=?")
89
+ .run(counter.parse(sequence + 1), streamId);
90
+ if (input.blobHashes !== undefined) {
91
+ const pin = db.prepare("INSERT INTO sync_outbox_blobs VALUES(?,?,?)");
92
+ for (const hash of new Set(input.blobHashes)) {
93
+ pin.run(streamId, sequence, z.string().regex(/^[a-f0-9]{64}$/).parse(hash));
94
+ }
95
+ }
96
+ db.exec("COMMIT");
97
+ return batch;
98
+ }
99
+ catch (error) {
100
+ db.exec("ROLLBACK");
101
+ throw error;
102
+ }
103
+ }
@@ -0,0 +1,27 @@
1
+ import { z } from "zod";
2
+ /** Source facts only: meta is daemon configuration/counters; FTS is a rebuildable index. */
3
+ export const targetTables = [
4
+ "skills", "sessions", "events", "loads", "eval_cases", "otel_spans",
5
+ "herdr_workspaces", "herdr_tabs", "herdr_panes", "pane_sessions",
6
+ "workflow_specs", "flows", "flow_steps", "load_candidates", "recording_clock",
7
+ ];
8
+ export const tableSchema = z.enum(targetTables);
9
+ export const scalarSchema = z.union([z.string(), z.number().finite(), z.null()]);
10
+ export const valuesSchema = z.record(z.string(), scalarSchema).readonly();
11
+ export const mutationSchema = z.strictObject({
12
+ table: tableSchema,
13
+ key: z.string(),
14
+ revision: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
15
+ operation: z.enum(["upsert", "delete"]),
16
+ sourceOrder: z.number().int().nullable(),
17
+ values: valuesSchema.nullable(),
18
+ beforeValues: valuesSchema.nullable().default(null),
19
+ }).readonly();
20
+ export class SyncError extends Error {
21
+ code;
22
+ constructor(code) {
23
+ super(code);
24
+ this.code = code;
25
+ this.name = "SyncError";
26
+ }
27
+ }