synapse-db 0.2.3 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/connectome.d.ts +11 -2
- package/dist/engine.d.ts +22 -3
- package/dist/index.d.ts +27 -0
- package/dist/index.js +1068 -100
- package/dist/serializer.d.ts +5 -0
- package/dist/types.d.ts +2 -0
- package/dist/wal/arena.d.ts +70 -0
- package/dist/wal/config.d.ts +52 -0
- package/dist/wal/index.d.ts +81 -0
- package/dist/wal/ringBuffer.d.ts +94 -0
- package/dist/wal/sqliteWalStore.d.ts +38 -0
- package/dist/wal/timeTravel.d.ts +34 -0
- package/package.json +2 -2
package/dist/index.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// @bun
|
|
2
2
|
// engine.ts
|
|
3
|
-
import { existsSync as
|
|
4
|
-
import { basename, join, resolve } from "path";
|
|
3
|
+
import { existsSync as existsSync3, mkdirSync as mkdirSync3, readdirSync, readFileSync as readFileSync2, unlinkSync as unlinkSync2, watch } from "fs";
|
|
4
|
+
import { basename, join as join2, resolve as resolve2 } from "path";
|
|
5
5
|
|
|
6
6
|
// connectome.ts
|
|
7
7
|
class SynapseConnectome {
|
|
@@ -12,6 +12,7 @@ class SynapseConnectome {
|
|
|
12
12
|
edgeMeta = new Map;
|
|
13
13
|
tokenInvertedIndex = new Map;
|
|
14
14
|
documentTokens = new Map;
|
|
15
|
+
dirtySearchNodes = new Set;
|
|
15
16
|
constructor() {}
|
|
16
17
|
static makeNodeId(collection, id) {
|
|
17
18
|
return `${collection}/${id}`;
|
|
@@ -24,10 +25,28 @@ class SynapseConnectome {
|
|
|
24
25
|
this.edgeMeta.clear();
|
|
25
26
|
this.tokenInvertedIndex.clear();
|
|
26
27
|
this.documentTokens.clear();
|
|
28
|
+
this.dirtySearchNodes.clear();
|
|
27
29
|
}
|
|
28
30
|
putDocument(doc) {
|
|
29
31
|
const nodeId = SynapseConnectome.makeNodeId(doc.collection, doc.id);
|
|
30
|
-
|
|
32
|
+
const links = doc.wikilinks;
|
|
33
|
+
const linksLen = links.length;
|
|
34
|
+
const existingOutSet = this.outgoingEdges.get(nodeId);
|
|
35
|
+
let skipEdgeRebuild = false;
|
|
36
|
+
if (existingOutSet && existingOutSet.size === 1 && linksLen === 1) {
|
|
37
|
+
const link = links[0];
|
|
38
|
+
const targetNodeId = SynapseConnectome.makeNodeId(link.targetCollection || doc.collection, link.targetId);
|
|
39
|
+
if (existingOutSet.has(targetNodeId)) {
|
|
40
|
+
const metaKey = `${nodeId}->${targetNodeId}`;
|
|
41
|
+
const existingMeta = this.edgeMeta.get(metaKey);
|
|
42
|
+
if (existingMeta && existingMeta.alias === link.alias) {
|
|
43
|
+
skipEdgeRebuild = true;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
if (!skipEdgeRebuild && existingOutSet) {
|
|
48
|
+
this.removeDocumentOutgoingEdges(nodeId);
|
|
49
|
+
}
|
|
31
50
|
this.documents.set(nodeId, doc);
|
|
32
51
|
let colSet = this.collectionDocIds.get(doc.collection);
|
|
33
52
|
if (!colSet) {
|
|
@@ -35,34 +54,40 @@ class SynapseConnectome {
|
|
|
35
54
|
this.collectionDocIds.set(doc.collection, colSet);
|
|
36
55
|
}
|
|
37
56
|
colSet.add(doc.id);
|
|
38
|
-
|
|
39
|
-
const targetNodeId = SynapseConnectome.makeNodeId(link.targetCollection || doc.collection, link.targetId);
|
|
57
|
+
if (!skipEdgeRebuild && linksLen > 0) {
|
|
40
58
|
let outSet = this.outgoingEdges.get(nodeId);
|
|
41
59
|
if (!outSet) {
|
|
42
60
|
outSet = new Set;
|
|
43
61
|
this.outgoingEdges.set(nodeId, outSet);
|
|
44
62
|
}
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
this.incomingEdges.
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
63
|
+
for (let i = 0;i < linksLen; i++) {
|
|
64
|
+
const link = links[i];
|
|
65
|
+
const targetNodeId = SynapseConnectome.makeNodeId(link.targetCollection || doc.collection, link.targetId);
|
|
66
|
+
outSet.add(targetNodeId);
|
|
67
|
+
let inSet = this.incomingEdges.get(targetNodeId);
|
|
68
|
+
if (!inSet) {
|
|
69
|
+
inSet = new Set;
|
|
70
|
+
this.incomingEdges.set(targetNodeId, inSet);
|
|
71
|
+
}
|
|
72
|
+
inSet.add(nodeId);
|
|
73
|
+
this.edgeMeta.set(`${nodeId}->${targetNodeId}`, {
|
|
74
|
+
fromNodeId: nodeId,
|
|
75
|
+
toNodeId: targetNodeId,
|
|
76
|
+
alias: link.alias
|
|
77
|
+
});
|
|
78
|
+
}
|
|
58
79
|
}
|
|
59
|
-
this.
|
|
80
|
+
this.dirtySearchNodes.add(nodeId);
|
|
60
81
|
}
|
|
61
82
|
removeDocument(collection, id) {
|
|
62
83
|
const nodeId = SynapseConnectome.makeNodeId(collection, id);
|
|
63
84
|
if (!this.documents.has(nodeId))
|
|
64
85
|
return false;
|
|
65
|
-
this.
|
|
86
|
+
this.dirtySearchNodes.delete(nodeId);
|
|
87
|
+
if (this.outgoingEdges.has(nodeId)) {
|
|
88
|
+
this.removeDocumentOutgoingEdges(nodeId);
|
|
89
|
+
}
|
|
90
|
+
this.removeDocumentSearchTokens(nodeId);
|
|
66
91
|
this.documents.delete(nodeId);
|
|
67
92
|
const colSet = this.collectionDocIds.get(collection);
|
|
68
93
|
if (colSet) {
|
|
@@ -73,34 +98,48 @@ class SynapseConnectome {
|
|
|
73
98
|
}
|
|
74
99
|
return true;
|
|
75
100
|
}
|
|
76
|
-
|
|
101
|
+
removeDocumentOutgoingEdges(nodeId) {
|
|
77
102
|
const outSet = this.outgoingEdges.get(nodeId);
|
|
78
|
-
if (outSet)
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
103
|
+
if (!outSet)
|
|
104
|
+
return;
|
|
105
|
+
for (const targetNodeId of outSet) {
|
|
106
|
+
const inSet = this.incomingEdges.get(targetNodeId);
|
|
107
|
+
if (inSet) {
|
|
108
|
+
inSet.delete(nodeId);
|
|
109
|
+
if (inSet.size === 0) {
|
|
110
|
+
this.incomingEdges.delete(targetNodeId);
|
|
86
111
|
}
|
|
87
|
-
this.edgeMeta.delete(`${nodeId}->${targetNodeId}`);
|
|
88
112
|
}
|
|
89
|
-
this.
|
|
113
|
+
this.edgeMeta.delete(`${nodeId}->${targetNodeId}`);
|
|
90
114
|
}
|
|
115
|
+
this.outgoingEdges.delete(nodeId);
|
|
116
|
+
}
|
|
117
|
+
removeDocumentSearchTokens(nodeId) {
|
|
91
118
|
const oldTokens = this.documentTokens.get(nodeId);
|
|
92
|
-
if (oldTokens)
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
119
|
+
if (!oldTokens)
|
|
120
|
+
return;
|
|
121
|
+
for (const token of oldTokens) {
|
|
122
|
+
const nodeSet = this.tokenInvertedIndex.get(token);
|
|
123
|
+
if (nodeSet) {
|
|
124
|
+
nodeSet.delete(nodeId);
|
|
125
|
+
if (nodeSet.size === 0) {
|
|
126
|
+
this.tokenInvertedIndex.delete(token);
|
|
100
127
|
}
|
|
101
128
|
}
|
|
102
|
-
this.documentTokens.delete(nodeId);
|
|
103
129
|
}
|
|
130
|
+
this.documentTokens.delete(nodeId);
|
|
131
|
+
}
|
|
132
|
+
flushDirtySearchIndex() {
|
|
133
|
+
if (this.dirtySearchNodes.size === 0)
|
|
134
|
+
return;
|
|
135
|
+
for (const nodeId of this.dirtySearchNodes) {
|
|
136
|
+
this.removeDocumentSearchTokens(nodeId);
|
|
137
|
+
const doc = this.documents.get(nodeId);
|
|
138
|
+
if (doc) {
|
|
139
|
+
this.indexDocumentText(nodeId, doc);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
this.dirtySearchNodes.clear();
|
|
104
143
|
}
|
|
105
144
|
getDocument(collection, id) {
|
|
106
145
|
const nodeId = SynapseConnectome.makeNodeId(collection, id);
|
|
@@ -139,6 +178,7 @@ class SynapseConnectome {
|
|
|
139
178
|
return results;
|
|
140
179
|
}
|
|
141
180
|
verifyGraphInvariants() {
|
|
181
|
+
this.flushDirtySearchIndex();
|
|
142
182
|
const brokenOutgoing = [];
|
|
143
183
|
const brokenIncoming = [];
|
|
144
184
|
const ghostTokens = [];
|
|
@@ -199,6 +239,7 @@ class SynapseConnectome {
|
|
|
199
239
|
return docs.slice(offset, offset + limit);
|
|
200
240
|
}
|
|
201
241
|
search(query, collectionFilter) {
|
|
242
|
+
this.flushDirtySearchIndex();
|
|
202
243
|
const tokens = this.tokenize(query);
|
|
203
244
|
if (tokens.length === 0)
|
|
204
245
|
return [];
|
|
@@ -279,32 +320,94 @@ class SynapseConnectome {
|
|
|
279
320
|
// serializer.ts
|
|
280
321
|
import { existsSync, mkdirSync, renameSync, unlinkSync, writeFileSync } from "fs";
|
|
281
322
|
import { dirname } from "path";
|
|
323
|
+
var EMPTY_WIKILINKS = [];
|
|
282
324
|
function extractWikilinks(text, defaultCollection = "") {
|
|
283
|
-
|
|
325
|
+
if (!text)
|
|
326
|
+
return EMPTY_WIKILINKS;
|
|
327
|
+
let openIdx = text.indexOf("[[");
|
|
328
|
+
if (openIdx === -1)
|
|
329
|
+
return EMPTY_WIKILINKS;
|
|
284
330
|
const links = [];
|
|
285
|
-
|
|
286
|
-
while (
|
|
287
|
-
const
|
|
288
|
-
if (
|
|
289
|
-
|
|
290
|
-
const
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
const parts = rawTarget.split("/");
|
|
295
|
-
targetCollection = parts[0].trim();
|
|
296
|
-
targetId = parts.slice(1).join("/").trim();
|
|
297
|
-
}
|
|
298
|
-
if (!targetId)
|
|
331
|
+
const textLen = text.length;
|
|
332
|
+
while (openIdx !== -1 && openIdx + 4 <= textLen) {
|
|
333
|
+
const closeIdx = text.indexOf("]]", openIdx + 2);
|
|
334
|
+
if (closeIdx === -1)
|
|
335
|
+
break;
|
|
336
|
+
const inner = text.slice(openIdx + 2, closeIdx);
|
|
337
|
+
const nestedOpen = inner.indexOf("[[");
|
|
338
|
+
if (nestedOpen !== -1) {
|
|
339
|
+
openIdx = openIdx + 2 + nestedOpen;
|
|
299
340
|
continue;
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
341
|
+
}
|
|
342
|
+
let rawTarget = inner;
|
|
343
|
+
let alias;
|
|
344
|
+
const pipeIdx = inner.indexOf("|");
|
|
345
|
+
if (pipeIdx !== -1) {
|
|
346
|
+
rawTarget = inner.slice(0, pipeIdx).trim();
|
|
347
|
+
const rawAlias = inner.slice(pipeIdx + 1).trim();
|
|
348
|
+
if (rawAlias.length > 0)
|
|
349
|
+
alias = rawAlias;
|
|
350
|
+
} else {
|
|
351
|
+
rawTarget = inner.trim();
|
|
352
|
+
}
|
|
353
|
+
if (rawTarget.length > 0) {
|
|
354
|
+
let targetCollection = defaultCollection;
|
|
355
|
+
let targetId = rawTarget;
|
|
356
|
+
const slashIdx = rawTarget.indexOf("/");
|
|
357
|
+
if (slashIdx !== -1) {
|
|
358
|
+
targetCollection = rawTarget.slice(0, slashIdx).trim();
|
|
359
|
+
targetId = rawTarget.slice(slashIdx + 1).trim();
|
|
360
|
+
}
|
|
361
|
+
if (targetId.length > 0) {
|
|
362
|
+
links.push({
|
|
363
|
+
raw: text.slice(openIdx, closeIdx + 2),
|
|
364
|
+
targetCollection,
|
|
365
|
+
targetId,
|
|
366
|
+
alias
|
|
367
|
+
});
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
openIdx = text.indexOf("[[", closeIdx + 2);
|
|
371
|
+
}
|
|
372
|
+
return links.length > 0 ? links : EMPTY_WIKILINKS;
|
|
373
|
+
}
|
|
374
|
+
function extractWikilinksFromDataAndBody(data, body, defaultCollection = "") {
|
|
375
|
+
let result = null;
|
|
376
|
+
if (data && typeof data === "object") {
|
|
377
|
+
for (const key in data) {
|
|
378
|
+
const val = data[key];
|
|
379
|
+
if (typeof val === "string" && val.includes("[[")) {
|
|
380
|
+
const extracted = extractWikilinks(val, defaultCollection);
|
|
381
|
+
if (extracted.length > 0) {
|
|
382
|
+
if (!result)
|
|
383
|
+
result = [];
|
|
384
|
+
for (let i = 0;i < extracted.length; i++)
|
|
385
|
+
result.push(extracted[i]);
|
|
386
|
+
}
|
|
387
|
+
} else if (typeof val === "object" && val !== null) {
|
|
388
|
+
const jsonStr = JSON.stringify(val);
|
|
389
|
+
if (jsonStr.includes("[[")) {
|
|
390
|
+
const extracted = extractWikilinks(jsonStr, defaultCollection);
|
|
391
|
+
if (extracted.length > 0) {
|
|
392
|
+
if (!result)
|
|
393
|
+
result = [];
|
|
394
|
+
for (let i = 0;i < extracted.length; i++)
|
|
395
|
+
result.push(extracted[i]);
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
if (body && body.includes("[[")) {
|
|
402
|
+
const bodyLinks = extractWikilinks(body, defaultCollection);
|
|
403
|
+
if (bodyLinks.length > 0) {
|
|
404
|
+
if (!result)
|
|
405
|
+
return bodyLinks;
|
|
406
|
+
for (let i = 0;i < bodyLinks.length; i++)
|
|
407
|
+
result.push(bodyLinks[i]);
|
|
408
|
+
}
|
|
306
409
|
}
|
|
307
|
-
return
|
|
410
|
+
return result ?? EMPTY_WIKILINKS;
|
|
308
411
|
}
|
|
309
412
|
function parseMarkdownFile(rawContent, defaultCollection = "") {
|
|
310
413
|
const normalizedRaw = rawContent.replace(/\r\n/g, `
|
|
@@ -505,14 +608,766 @@ function atomicWriteFile(targetPath, content) {
|
|
|
505
608
|
}
|
|
506
609
|
}
|
|
507
610
|
|
|
611
|
+
// wal/index.ts
|
|
612
|
+
import { join } from "path";
|
|
613
|
+
|
|
614
|
+
// wal/config.ts
|
|
615
|
+
var WAL_CONFIG_DEFAULTS = {
|
|
616
|
+
RING_BUFFER_CAPACITY: Number(process.env.SYNAPSE_WAL_RING_CAPACITY) || 4096,
|
|
617
|
+
ARENA_INITIAL_CAPACITY: Number(process.env.SYNAPSE_WAL_ARENA_CAPACITY) || 256,
|
|
618
|
+
ARENA_MAX_CAPACITY: Number(process.env.SYNAPSE_WAL_ARENA_MAX_CAPACITY) || 65536,
|
|
619
|
+
SQLITE_WAL_FILENAME: process.env.SYNAPSE_WAL_SQLITE_FILENAME || "synapse_causal_wal.sqlite",
|
|
620
|
+
SQLITE_MMAP_SIZE_BYTES: Number(process.env.SYNAPSE_WAL_MMAP_BYTES) || 268435456,
|
|
621
|
+
SQLITE_BUSY_TIMEOUT_MS: Number(process.env.SYNAPSE_WAL_BUSY_TIMEOUT_MS) || 5000,
|
|
622
|
+
SQLITE_SYNC_MODE: process.env.SYNAPSE_WAL_SYNC_MODE || "NORMAL"
|
|
623
|
+
};
|
|
624
|
+
var CausalWalEventType;
|
|
625
|
+
((CausalWalEventType) => {
|
|
626
|
+
CausalWalEventType["DOC_UPSERT"] = "DOC_UPSERT";
|
|
627
|
+
CausalWalEventType["DOC_DELETE"] = "DOC_DELETE";
|
|
628
|
+
CausalWalEventType["EDGE_LINK"] = "EDGE_LINK";
|
|
629
|
+
CausalWalEventType["EDGE_UNLINK"] = "EDGE_UNLINK";
|
|
630
|
+
CausalWalEventType["COMPILER_METRIC"] = "COMPILER_METRIC";
|
|
631
|
+
CausalWalEventType["CHECKPOINT_MARK"] = "CHECKPOINT_MARK";
|
|
632
|
+
})(CausalWalEventType ||= {});
|
|
633
|
+
|
|
634
|
+
// wal/ringBuffer.ts
|
|
635
|
+
class TemporalRingBuffer {
|
|
636
|
+
capacity;
|
|
637
|
+
isPowerOfTwo;
|
|
638
|
+
bitMask;
|
|
639
|
+
slots;
|
|
640
|
+
lsnBuffer;
|
|
641
|
+
tsBuffer;
|
|
642
|
+
head = 0;
|
|
643
|
+
tail = 0;
|
|
644
|
+
count = 0;
|
|
645
|
+
nextLsn = 1;
|
|
646
|
+
overwrittenCount = 0;
|
|
647
|
+
constructor(capacity = WAL_CONFIG_DEFAULTS.RING_BUFFER_CAPACITY) {
|
|
648
|
+
if (!Number.isInteger(capacity) || capacity <= 0) {
|
|
649
|
+
throw new Error(`[TemporalRingBuffer] \uC720\uD6A8\uD558\uC9C0 \uC54A\uC740 \uB9C1 \uBC84\uD37C \uC6A9\uB7C9\uC785\uB2C8\uB2E4: ${capacity}`);
|
|
650
|
+
}
|
|
651
|
+
this.capacity = capacity;
|
|
652
|
+
this.isPowerOfTwo = (capacity & capacity - 1) === 0;
|
|
653
|
+
this.bitMask = capacity - 1;
|
|
654
|
+
this.slots = new Array(capacity);
|
|
655
|
+
this.lsnBuffer = new Float64Array(capacity);
|
|
656
|
+
this.tsBuffer = new Float64Array(capacity);
|
|
657
|
+
}
|
|
658
|
+
wrapIndex(idx) {
|
|
659
|
+
return this.isPowerOfTwo ? idx & this.bitMask : idx % this.capacity;
|
|
660
|
+
}
|
|
661
|
+
append(entryInput) {
|
|
662
|
+
const lsn = this.nextLsn++;
|
|
663
|
+
const timestampMs = entryInput.timestampMs ?? Date.now();
|
|
664
|
+
const entry = {
|
|
665
|
+
lsn,
|
|
666
|
+
timestampMs,
|
|
667
|
+
eventType: entryInput.eventType,
|
|
668
|
+
collection: entryInput.collection,
|
|
669
|
+
docId: entryInput.docId,
|
|
670
|
+
prevDoc: entryInput.prevDoc,
|
|
671
|
+
nextDoc: entryInput.nextDoc,
|
|
672
|
+
edgesAdded: entryInput.edgesAdded,
|
|
673
|
+
edgesRemoved: entryInput.edgesRemoved,
|
|
674
|
+
compilerMetric: entryInput.compilerMetric
|
|
675
|
+
};
|
|
676
|
+
this.writeRawEntry(entry);
|
|
677
|
+
return entry;
|
|
678
|
+
}
|
|
679
|
+
appendRecovered(entry) {
|
|
680
|
+
if (entry.lsn >= this.nextLsn) {
|
|
681
|
+
this.nextLsn = entry.lsn + 1;
|
|
682
|
+
}
|
|
683
|
+
this.writeRawEntry(entry);
|
|
684
|
+
}
|
|
685
|
+
writeRawEntry(entry) {
|
|
686
|
+
const t = this.tail;
|
|
687
|
+
this.slots[t] = entry;
|
|
688
|
+
this.lsnBuffer[t] = entry.lsn;
|
|
689
|
+
this.tsBuffer[t] = entry.timestampMs;
|
|
690
|
+
const nextTail = this.isPowerOfTwo ? t + 1 & this.bitMask : (t + 1) % this.capacity;
|
|
691
|
+
this.tail = nextTail;
|
|
692
|
+
if (this.count === this.capacity) {
|
|
693
|
+
this.head = nextTail;
|
|
694
|
+
this.overwrittenCount++;
|
|
695
|
+
} else {
|
|
696
|
+
this.count++;
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
getByLsn(targetLsn) {
|
|
700
|
+
if (this.count === 0)
|
|
701
|
+
return null;
|
|
702
|
+
const oldestLsn = this.lsnBuffer[this.head];
|
|
703
|
+
const newestIdx = this.wrapIndex(this.tail - 1 + this.capacity);
|
|
704
|
+
const newestLsn = this.lsnBuffer[newestIdx];
|
|
705
|
+
if (targetLsn < oldestLsn || targetLsn > newestLsn) {
|
|
706
|
+
return null;
|
|
707
|
+
}
|
|
708
|
+
const offset = targetLsn - oldestLsn;
|
|
709
|
+
if (offset < this.count) {
|
|
710
|
+
const candidateIdx = this.wrapIndex(this.head + offset);
|
|
711
|
+
if (this.lsnBuffer[candidateIdx] === targetLsn) {
|
|
712
|
+
return this.slots[candidateIdx] || null;
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
for (let i = 0;i < this.count; i++) {
|
|
716
|
+
const idx = this.wrapIndex(this.head + i);
|
|
717
|
+
if (this.lsnBuffer[idx] === targetLsn) {
|
|
718
|
+
return this.slots[idx] || null;
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
return null;
|
|
722
|
+
}
|
|
723
|
+
forEachReverse(visitor) {
|
|
724
|
+
for (let i = this.count - 1;i >= 0; i--) {
|
|
725
|
+
const idx = (this.head + i) % this.capacity;
|
|
726
|
+
const entry = this.slots[idx];
|
|
727
|
+
if (entry) {
|
|
728
|
+
const shouldContinue = visitor(entry);
|
|
729
|
+
if (shouldContinue === false)
|
|
730
|
+
break;
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
}
|
|
734
|
+
toArray() {
|
|
735
|
+
const result = new Array(this.count);
|
|
736
|
+
for (let i = 0;i < this.count; i++) {
|
|
737
|
+
const idx = (this.head + i) % this.capacity;
|
|
738
|
+
result[i] = this.slots[idx];
|
|
739
|
+
}
|
|
740
|
+
return result;
|
|
741
|
+
}
|
|
742
|
+
getEntriesSinceLsn(afterLsn) {
|
|
743
|
+
const result = [];
|
|
744
|
+
for (let i = 0;i < this.count; i++) {
|
|
745
|
+
const idx = (this.head + i) % this.capacity;
|
|
746
|
+
const entry = this.slots[idx];
|
|
747
|
+
if (entry && entry.lsn > afterLsn) {
|
|
748
|
+
result.push(entry);
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
return result;
|
|
752
|
+
}
|
|
753
|
+
getCurrentLsn() {
|
|
754
|
+
return this.nextLsn - 1;
|
|
755
|
+
}
|
|
756
|
+
getOldestLsn() {
|
|
757
|
+
if (this.count === 0)
|
|
758
|
+
return 0;
|
|
759
|
+
return this.lsnBuffer[this.head];
|
|
760
|
+
}
|
|
761
|
+
getStats() {
|
|
762
|
+
return {
|
|
763
|
+
capacity: this.capacity,
|
|
764
|
+
size: this.count,
|
|
765
|
+
currentLsn: this.getCurrentLsn(),
|
|
766
|
+
oldestLsn: this.getOldestLsn(),
|
|
767
|
+
overwrittenCount: this.overwrittenCount,
|
|
768
|
+
utilizationRatio: Number((this.count / this.capacity).toFixed(4))
|
|
769
|
+
};
|
|
770
|
+
}
|
|
771
|
+
clear() {
|
|
772
|
+
this.slots.fill(undefined);
|
|
773
|
+
this.lsnBuffer.fill(0);
|
|
774
|
+
this.tsBuffer.fill(0);
|
|
775
|
+
this.head = 0;
|
|
776
|
+
this.tail = 0;
|
|
777
|
+
this.count = 0;
|
|
778
|
+
this.nextLsn = 1;
|
|
779
|
+
this.overwrittenCount = 0;
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
// wal/arena.ts
|
|
784
|
+
class CausalArenaAllocator {
|
|
785
|
+
pool;
|
|
786
|
+
cursor = 0;
|
|
787
|
+
inTransaction = false;
|
|
788
|
+
totalCommitted = 0;
|
|
789
|
+
totalRolledBack = 0;
|
|
790
|
+
totalSlotReuses = 0;
|
|
791
|
+
constructor(initialCapacity = WAL_CONFIG_DEFAULTS.ARENA_INITIAL_CAPACITY) {
|
|
792
|
+
this.pool = new Array(initialCapacity);
|
|
793
|
+
for (let i = 0;i < initialCapacity; i++) {
|
|
794
|
+
this.pool[i] = this.createEmptySlot();
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
createEmptySlot() {
|
|
798
|
+
return {
|
|
799
|
+
eventType: "DOC_UPSERT" /* DOC_UPSERT */,
|
|
800
|
+
collection: "",
|
|
801
|
+
docId: "",
|
|
802
|
+
prevDoc: null,
|
|
803
|
+
nextDoc: null,
|
|
804
|
+
edgesAdded: [],
|
|
805
|
+
edgesRemoved: [],
|
|
806
|
+
rawMarkdown: null,
|
|
807
|
+
targetDiskPath: null,
|
|
808
|
+
compilerMetric: undefined
|
|
809
|
+
};
|
|
810
|
+
}
|
|
811
|
+
beginScope() {
|
|
812
|
+
if (this.inTransaction) {
|
|
813
|
+
throw new Error("[CausalArenaAllocator] \uC911\uCCA9 \uD2B8\uB79C\uC7AD\uC158\uC740 \uC774\uBBF8 \uD65C\uC131\uD654\uB41C \uC2A4\uCF54\uD504\uAC00 \uC874\uC7AC\uD569\uB2C8\uB2E4.");
|
|
814
|
+
}
|
|
815
|
+
this.inTransaction = true;
|
|
816
|
+
this.cursor = 0;
|
|
817
|
+
}
|
|
818
|
+
isScopeActive() {
|
|
819
|
+
return this.inTransaction;
|
|
820
|
+
}
|
|
821
|
+
allocSlot(eventType, collection, docId, prevDoc, nextDoc, edgesAdded, edgesRemoved, rawMarkdown = null, targetDiskPath = null, compilerMetric) {
|
|
822
|
+
if (this.cursor >= this.pool.length) {
|
|
823
|
+
if (this.pool.length >= WAL_CONFIG_DEFAULTS.ARENA_MAX_CAPACITY) {
|
|
824
|
+
throw new Error(`[CausalArenaAllocator] \uC544\uB808\uB098 \uCD5C\uB300 \uC6A9\uB7C9 \uC0C1\uD55C\uC120(${WAL_CONFIG_DEFAULTS.ARENA_MAX_CAPACITY})\uC744 \uCD08\uACFC\uD588\uC2B5\uB2C8\uB2E4.`);
|
|
825
|
+
}
|
|
826
|
+
const expandCount = Math.min(this.pool.length, WAL_CONFIG_DEFAULTS.ARENA_MAX_CAPACITY - this.pool.length);
|
|
827
|
+
for (let i = 0;i < expandCount; i++) {
|
|
828
|
+
this.pool.push(this.createEmptySlot());
|
|
829
|
+
}
|
|
830
|
+
} else {
|
|
831
|
+
this.totalSlotReuses++;
|
|
832
|
+
}
|
|
833
|
+
const slot = this.pool[this.cursor++];
|
|
834
|
+
slot.eventType = eventType;
|
|
835
|
+
slot.collection = collection;
|
|
836
|
+
slot.docId = docId;
|
|
837
|
+
slot.prevDoc = prevDoc;
|
|
838
|
+
slot.nextDoc = nextDoc;
|
|
839
|
+
slot.edgesAdded = edgesAdded;
|
|
840
|
+
slot.edgesRemoved = edgesRemoved;
|
|
841
|
+
slot.rawMarkdown = rawMarkdown;
|
|
842
|
+
slot.targetDiskPath = targetDiskPath;
|
|
843
|
+
slot.compilerMetric = compilerMetric;
|
|
844
|
+
return slot;
|
|
845
|
+
}
|
|
846
|
+
getActiveSlots() {
|
|
847
|
+
return this.pool.slice(0, this.cursor);
|
|
848
|
+
}
|
|
849
|
+
commitAndReset() {
|
|
850
|
+
this.clearReferencesUpToCursor();
|
|
851
|
+
this.cursor = 0;
|
|
852
|
+
this.inTransaction = false;
|
|
853
|
+
this.totalCommitted++;
|
|
854
|
+
}
|
|
855
|
+
rollbackAndReset() {
|
|
856
|
+
this.clearReferencesUpToCursor();
|
|
857
|
+
this.cursor = 0;
|
|
858
|
+
this.inTransaction = false;
|
|
859
|
+
this.totalRolledBack++;
|
|
860
|
+
}
|
|
861
|
+
clearReferencesUpToCursor() {
|
|
862
|
+
for (let i = 0;i < this.cursor; i++) {
|
|
863
|
+
const slot = this.pool[i];
|
|
864
|
+
slot.prevDoc = null;
|
|
865
|
+
slot.nextDoc = null;
|
|
866
|
+
slot.rawMarkdown = null;
|
|
867
|
+
slot.targetDiskPath = null;
|
|
868
|
+
slot.compilerMetric = undefined;
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
getStats() {
|
|
872
|
+
return {
|
|
873
|
+
poolCapacity: this.pool.length,
|
|
874
|
+
activeCursor: this.cursor,
|
|
875
|
+
totalTransactionsCommitted: this.totalCommitted,
|
|
876
|
+
totalTransactionsRolledBack: this.totalRolledBack,
|
|
877
|
+
totalSlotReuses: this.totalSlotReuses
|
|
878
|
+
};
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
// wal/sqliteWalStore.ts
|
|
883
|
+
import { existsSync as existsSync2, mkdirSync as mkdirSync2 } from "fs";
|
|
884
|
+
import { dirname as dirname2, resolve } from "path";
|
|
885
|
+
import { Database } from "bun:sqlite";
|
|
886
|
+
class SqliteCausalWalStore {
|
|
887
|
+
syncMode;
|
|
888
|
+
dbPath;
|
|
889
|
+
sqlite = null;
|
|
890
|
+
insertStmt = null;
|
|
891
|
+
batchTx = null;
|
|
892
|
+
lastFlushedLsn = 0;
|
|
893
|
+
constructor(dbPath, syncMode = WAL_CONFIG_DEFAULTS.SQLITE_SYNC_MODE) {
|
|
894
|
+
this.syncMode = syncMode;
|
|
895
|
+
this.dbPath = resolve(dbPath);
|
|
896
|
+
}
|
|
897
|
+
init() {
|
|
898
|
+
if (this.sqlite)
|
|
899
|
+
return;
|
|
900
|
+
try {
|
|
901
|
+
const parentDir = dirname2(this.dbPath);
|
|
902
|
+
if (!existsSync2(parentDir)) {
|
|
903
|
+
mkdirSync2(parentDir, { recursive: true });
|
|
904
|
+
}
|
|
905
|
+
this.sqlite = new Database(this.dbPath, { create: true });
|
|
906
|
+
this.sqlite.exec(`PRAGMA journal_mode = WAL;`);
|
|
907
|
+
this.sqlite.exec(`PRAGMA synchronous = ${this.syncMode};`);
|
|
908
|
+
this.sqlite.exec(`PRAGMA temp_store = MEMORY;`);
|
|
909
|
+
this.sqlite.exec(`PRAGMA mmap_size = ${WAL_CONFIG_DEFAULTS.SQLITE_MMAP_SIZE_BYTES};`);
|
|
910
|
+
this.sqlite.exec(`PRAGMA busy_timeout = ${WAL_CONFIG_DEFAULTS.SQLITE_BUSY_TIMEOUT_MS};`);
|
|
911
|
+
this.sqlite.exec(`
|
|
912
|
+
CREATE TABLE IF NOT EXISTS causal_wal_log (
|
|
913
|
+
lsn INTEGER PRIMARY KEY,
|
|
914
|
+
timestamp_ms INTEGER NOT NULL,
|
|
915
|
+
event_type TEXT NOT NULL,
|
|
916
|
+
collection TEXT NOT NULL,
|
|
917
|
+
doc_id TEXT NOT NULL,
|
|
918
|
+
prev_doc_json TEXT,
|
|
919
|
+
next_doc_json TEXT,
|
|
920
|
+
edges_added_json TEXT NOT NULL,
|
|
921
|
+
edges_removed_json TEXT NOT NULL,
|
|
922
|
+
compiler_metric_json TEXT
|
|
923
|
+
);
|
|
924
|
+
CREATE INDEX IF NOT EXISTS idx_causal_wal_ts ON causal_wal_log(timestamp_ms);
|
|
925
|
+
CREATE INDEX IF NOT EXISTS idx_causal_wal_doc ON causal_wal_log(collection, doc_id);
|
|
926
|
+
`);
|
|
927
|
+
const maxRow = this.sqlite.query(`SELECT MAX(lsn) as max_lsn FROM causal_wal_log`).get();
|
|
928
|
+
this.lastFlushedLsn = maxRow?.max_lsn ?? 0;
|
|
929
|
+
const stmt = this.sqlite.prepare(`
|
|
930
|
+
INSERT OR REPLACE INTO causal_wal_log (
|
|
931
|
+
lsn, timestamp_ms, event_type, collection, doc_id,
|
|
932
|
+
prev_doc_json, next_doc_json, edges_added_json, edges_removed_json, compiler_metric_json
|
|
933
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
934
|
+
`);
|
|
935
|
+
this.insertStmt = stmt;
|
|
936
|
+
this.batchTx = this.sqlite.transaction((items) => {
|
|
937
|
+
let written = 0;
|
|
938
|
+
for (let i = 0;i < items.length; i++) {
|
|
939
|
+
const item = items[i];
|
|
940
|
+
if (item.lsn <= this.lastFlushedLsn)
|
|
941
|
+
continue;
|
|
942
|
+
stmt.run(item.lsn, item.timestampMs, item.eventType, item.collection, item.docId, item.prevDoc ? JSON.stringify(item.prevDoc) : null, item.nextDoc ? JSON.stringify(item.nextDoc) : null, item.edgesAdded.length === 0 ? "[]" : JSON.stringify(item.edgesAdded), item.edgesRemoved.length === 0 ? "[]" : JSON.stringify(item.edgesRemoved), item.compilerMetric ? JSON.stringify(item.compilerMetric) : null);
|
|
943
|
+
if (item.lsn > this.lastFlushedLsn) {
|
|
944
|
+
this.lastFlushedLsn = item.lsn;
|
|
945
|
+
}
|
|
946
|
+
written++;
|
|
947
|
+
}
|
|
948
|
+
return written;
|
|
949
|
+
});
|
|
950
|
+
} catch (err) {
|
|
951
|
+
throw new Error(`[SqliteCausalWalStore] SQLite WAL \uCD08\uAE30\uD654 \uC2E4\uD328 (${this.dbPath}): ${err?.message || err}`);
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
flushEntries(entries) {
|
|
955
|
+
if (!this.sqlite || !this.batchTx || entries.length === 0)
|
|
956
|
+
return 0;
|
|
957
|
+
try {
|
|
958
|
+
return this.batchTx(entries);
|
|
959
|
+
} catch (err) {
|
|
960
|
+
throw new Error(`[SqliteCausalWalStore] WAL \uC5D4\uD2B8\uB9AC \uBC30\uCE58 \uD50C\uB7EC\uC2DC \uC2E4\uD328: ${err?.message || err}`);
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
loadAllEntries() {
|
|
964
|
+
if (!this.sqlite)
|
|
965
|
+
return [];
|
|
966
|
+
try {
|
|
967
|
+
const rows = this.sqlite.query(`SELECT * FROM causal_wal_log ORDER BY lsn ASC`).all();
|
|
968
|
+
return rows.map((row) => ({
|
|
969
|
+
lsn: row.lsn,
|
|
970
|
+
timestampMs: row.timestamp_ms,
|
|
971
|
+
eventType: row.event_type,
|
|
972
|
+
collection: row.collection,
|
|
973
|
+
docId: row.doc_id,
|
|
974
|
+
prevDoc: row.prev_doc_json ? JSON.parse(row.prev_doc_json) : null,
|
|
975
|
+
nextDoc: row.next_doc_json ? JSON.parse(row.next_doc_json) : null,
|
|
976
|
+
edgesAdded: JSON.parse(row.edges_added_json),
|
|
977
|
+
edgesRemoved: JSON.parse(row.edges_removed_json),
|
|
978
|
+
compilerMetric: row.compiler_metric_json ? JSON.parse(row.compiler_metric_json) : undefined
|
|
979
|
+
}));
|
|
980
|
+
} catch (err) {
|
|
981
|
+
throw new Error(`[SqliteCausalWalStore] WAL \uBCF5\uAD6C \uB85C\uADF8 \uB85C\uB4DC \uC2E4\uD328: ${err?.message || err}`);
|
|
982
|
+
}
|
|
983
|
+
}
|
|
984
|
+
getJournalMode() {
|
|
985
|
+
if (!this.sqlite)
|
|
986
|
+
return "closed";
|
|
987
|
+
const row = this.sqlite.query(`PRAGMA journal_mode;`).get();
|
|
988
|
+
return row?.journal_mode ?? "unknown";
|
|
989
|
+
}
|
|
990
|
+
getLastFlushedLsn() {
|
|
991
|
+
return this.lastFlushedLsn;
|
|
992
|
+
}
|
|
993
|
+
getPersistedCount() {
|
|
994
|
+
if (!this.sqlite)
|
|
995
|
+
return 0;
|
|
996
|
+
const row = this.sqlite.query(`SELECT COUNT(*) as cnt FROM causal_wal_log`).get();
|
|
997
|
+
return row?.cnt ?? 0;
|
|
998
|
+
}
|
|
999
|
+
close() {
|
|
1000
|
+
if (!this.sqlite)
|
|
1001
|
+
return;
|
|
1002
|
+
try {
|
|
1003
|
+
this.insertStmt?.finalize();
|
|
1004
|
+
} catch {}
|
|
1005
|
+
this.insertStmt = null;
|
|
1006
|
+
this.batchTx = null;
|
|
1007
|
+
try {
|
|
1008
|
+
this.sqlite.exec(`PRAGMA wal_checkpoint(TRUNCATE);`);
|
|
1009
|
+
} catch {}
|
|
1010
|
+
try {
|
|
1011
|
+
this.sqlite.close();
|
|
1012
|
+
} catch {}
|
|
1013
|
+
this.sqlite = null;
|
|
1014
|
+
}
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
// wal/timeTravel.ts
|
|
1018
|
+
class CausalTimeTravelEngine {
|
|
1019
|
+
static asOfLsn(currentConnectome, ringBuffer, targetLsn) {
|
|
1020
|
+
const currentLsn = ringBuffer.getCurrentLsn();
|
|
1021
|
+
if (targetLsn < 0) {
|
|
1022
|
+
throw new Error(`[CausalTimeTravelEngine] \uC720\uD6A8\uD558\uC9C0 \uC54A\uC740 \uBAA9\uD45C LSN\uC785\uB2C8\uB2E4: ${targetLsn}`);
|
|
1023
|
+
}
|
|
1024
|
+
const docMap = new Map;
|
|
1025
|
+
for (const doc of currentConnectome.getAllDocuments()) {
|
|
1026
|
+
const nodeId = SynapseConnectome.makeNodeId(doc.collection, doc.id);
|
|
1027
|
+
docMap.set(nodeId, doc);
|
|
1028
|
+
}
|
|
1029
|
+
if (targetLsn < currentLsn) {
|
|
1030
|
+
ringBuffer.forEachReverse((entry) => {
|
|
1031
|
+
if (entry.lsn <= targetLsn) {
|
|
1032
|
+
return false;
|
|
1033
|
+
}
|
|
1034
|
+
if (entry.eventType === "DOC_UPSERT" /* DOC_UPSERT */ || entry.eventType === "DOC_DELETE" /* DOC_DELETE */) {
|
|
1035
|
+
const nodeId = SynapseConnectome.makeNodeId(entry.collection, entry.docId);
|
|
1036
|
+
if (entry.prevDoc !== null) {
|
|
1037
|
+
docMap.set(nodeId, entry.prevDoc);
|
|
1038
|
+
} else {
|
|
1039
|
+
docMap.delete(nodeId);
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
return true;
|
|
1043
|
+
});
|
|
1044
|
+
}
|
|
1045
|
+
const snapshotConnectome = new SynapseConnectome;
|
|
1046
|
+
for (const doc of docMap.values()) {
|
|
1047
|
+
snapshotConnectome.putDocument(doc);
|
|
1048
|
+
}
|
|
1049
|
+
return snapshotConnectome;
|
|
1050
|
+
}
|
|
1051
|
+
static asOf(currentConnectome, ringBuffer, timestampMsOrIso) {
|
|
1052
|
+
const targetMs = typeof timestampMsOrIso === "number" ? timestampMsOrIso : new Date(timestampMsOrIso).getTime();
|
|
1053
|
+
if (Number.isNaN(targetMs)) {
|
|
1054
|
+
throw new Error(`[CausalTimeTravelEngine] \uC720\uD6A8\uD558\uC9C0 \uC54A\uC740 \uD0C0\uC784\uC2A4\uD0EC\uD504 \uD615\uC2DD\uC785\uB2C8\uB2E4: ${timestampMsOrIso}`);
|
|
1055
|
+
}
|
|
1056
|
+
let resolvedLsn = 0;
|
|
1057
|
+
ringBuffer.forEachReverse((entry) => {
|
|
1058
|
+
if (entry.timestampMs <= targetMs) {
|
|
1059
|
+
resolvedLsn = entry.lsn;
|
|
1060
|
+
return false;
|
|
1061
|
+
}
|
|
1062
|
+
return true;
|
|
1063
|
+
});
|
|
1064
|
+
return CausalTimeTravelEngine.asOfLsn(currentConnectome, ringBuffer, resolvedLsn);
|
|
1065
|
+
}
|
|
1066
|
+
static diffBetweenLsn(ringBuffer, fromLsn, toLsn) {
|
|
1067
|
+
const addedSet = new Set;
|
|
1068
|
+
const modifiedSet = new Set;
|
|
1069
|
+
const deletedSet = new Set;
|
|
1070
|
+
const linkedSet = new Set;
|
|
1071
|
+
const unlinkedSet = new Set;
|
|
1072
|
+
const compilerMetrics = [];
|
|
1073
|
+
const entries = ringBuffer.getEntriesSinceLsn(fromLsn);
|
|
1074
|
+
for (const entry of entries) {
|
|
1075
|
+
if (entry.lsn > toLsn)
|
|
1076
|
+
break;
|
|
1077
|
+
const nodeId = SynapseConnectome.makeNodeId(entry.collection, entry.docId);
|
|
1078
|
+
if (entry.eventType === "DOC_UPSERT" /* DOC_UPSERT */) {
|
|
1079
|
+
deletedSet.delete(nodeId);
|
|
1080
|
+
if (entry.prevDoc === null) {
|
|
1081
|
+
addedSet.add(nodeId);
|
|
1082
|
+
} else if (!addedSet.has(nodeId)) {
|
|
1083
|
+
modifiedSet.add(nodeId);
|
|
1084
|
+
}
|
|
1085
|
+
} else if (entry.eventType === "DOC_DELETE" /* DOC_DELETE */) {
|
|
1086
|
+
if (addedSet.has(nodeId)) {
|
|
1087
|
+
addedSet.delete(nodeId);
|
|
1088
|
+
} else {
|
|
1089
|
+
modifiedSet.delete(nodeId);
|
|
1090
|
+
deletedSet.add(nodeId);
|
|
1091
|
+
}
|
|
1092
|
+
} else if (entry.eventType === "COMPILER_METRIC" /* COMPILER_METRIC */ && entry.compilerMetric) {
|
|
1093
|
+
compilerMetrics.push(entry.compilerMetric);
|
|
1094
|
+
}
|
|
1095
|
+
for (const edge of entry.edgesAdded) {
|
|
1096
|
+
const edgeKey = `${nodeId}->${edge}`;
|
|
1097
|
+
unlinkedSet.delete(edgeKey);
|
|
1098
|
+
linkedSet.add(edgeKey);
|
|
1099
|
+
}
|
|
1100
|
+
for (const edge of entry.edgesRemoved) {
|
|
1101
|
+
const edgeKey = `${nodeId}->${edge}`;
|
|
1102
|
+
linkedSet.delete(edgeKey);
|
|
1103
|
+
unlinkedSet.add(edgeKey);
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
return {
|
|
1107
|
+
fromLsn,
|
|
1108
|
+
toLsn,
|
|
1109
|
+
addedDocuments: Array.from(addedSet),
|
|
1110
|
+
modifiedDocuments: Array.from(modifiedSet),
|
|
1111
|
+
deletedDocuments: Array.from(deletedSet),
|
|
1112
|
+
edgesLinked: Array.from(linkedSet),
|
|
1113
|
+
edgesUnlinked: Array.from(unlinkedSet),
|
|
1114
|
+
compilerMetrics
|
|
1115
|
+
};
|
|
1116
|
+
}
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
// wal/index.ts
|
|
1120
|
+
var EMPTY_EDGES = [];
|
|
1121
|
+
var EMPTY_EDGE_DIFF = { edgesAdded: EMPTY_EDGES, edgesRemoved: EMPTY_EDGES };
|
|
1122
|
+
|
|
1123
|
+
class CausalWalEngine {
|
|
1124
|
+
ringBuffer;
|
|
1125
|
+
arena;
|
|
1126
|
+
sqliteStore;
|
|
1127
|
+
constructor(vaultDir, config2) {
|
|
1128
|
+
this.ringBuffer = new TemporalRingBuffer(config2?.ringCapacity ?? WAL_CONFIG_DEFAULTS.RING_BUFFER_CAPACITY);
|
|
1129
|
+
this.arena = new CausalArenaAllocator(config2?.arenaCapacity ?? WAL_CONFIG_DEFAULTS.ARENA_INITIAL_CAPACITY);
|
|
1130
|
+
if (config2?.persistSqlite || config2?.sqlitePath) {
|
|
1131
|
+
const targetSqlitePath = config2.sqlitePath || join(vaultDir, WAL_CONFIG_DEFAULTS.SQLITE_WAL_FILENAME);
|
|
1132
|
+
this.sqliteStore = new SqliteCausalWalStore(targetSqlitePath, config2.syncMode ?? WAL_CONFIG_DEFAULTS.SQLITE_SYNC_MODE);
|
|
1133
|
+
} else {
|
|
1134
|
+
this.sqliteStore = null;
|
|
1135
|
+
}
|
|
1136
|
+
}
|
|
1137
|
+
initAndRecover(connectome) {
|
|
1138
|
+
if (!this.sqliteStore)
|
|
1139
|
+
return 0;
|
|
1140
|
+
this.sqliteStore.init();
|
|
1141
|
+
const persistedEntries = this.sqliteStore.loadAllEntries();
|
|
1142
|
+
let recoveredCount = 0;
|
|
1143
|
+
for (const entry of persistedEntries) {
|
|
1144
|
+
this.ringBuffer.appendRecovered(entry);
|
|
1145
|
+
if (entry.eventType === "DOC_UPSERT" /* DOC_UPSERT */ && entry.nextDoc) {
|
|
1146
|
+
connectome.putDocument(entry.nextDoc);
|
|
1147
|
+
recoveredCount++;
|
|
1148
|
+
} else if (entry.eventType === "DOC_DELETE" /* DOC_DELETE */) {
|
|
1149
|
+
connectome.removeDocument(entry.collection, entry.docId);
|
|
1150
|
+
recoveredCount++;
|
|
1151
|
+
}
|
|
1152
|
+
}
|
|
1153
|
+
return recoveredCount;
|
|
1154
|
+
}
|
|
1155
|
+
static computeEdgeDiff(prevDoc, nextDoc) {
|
|
1156
|
+
const prevLinks = prevDoc ? prevDoc.wikilinks : EMPTY_EDGES;
|
|
1157
|
+
const nextLinks = nextDoc ? nextDoc.wikilinks : EMPTY_EDGES;
|
|
1158
|
+
const prevLen = prevLinks.length;
|
|
1159
|
+
const nextLen = nextLinks.length;
|
|
1160
|
+
if (prevLen === 0 && nextLen === 0) {
|
|
1161
|
+
return EMPTY_EDGE_DIFF;
|
|
1162
|
+
}
|
|
1163
|
+
if (prevLen === 0 && nextLen === 1 && nextDoc) {
|
|
1164
|
+
const l = nextDoc.wikilinks[0];
|
|
1165
|
+
return {
|
|
1166
|
+
edgesAdded: [SynapseConnectome.makeNodeId(l.targetCollection || nextDoc.collection, l.targetId)],
|
|
1167
|
+
edgesRemoved: EMPTY_EDGES
|
|
1168
|
+
};
|
|
1169
|
+
}
|
|
1170
|
+
if (prevLen === 1 && nextLen === 1 && prevDoc && nextDoc) {
|
|
1171
|
+
const pl = prevDoc.wikilinks[0];
|
|
1172
|
+
const nl = nextDoc.wikilinks[0];
|
|
1173
|
+
const pTarget = SynapseConnectome.makeNodeId(pl.targetCollection || prevDoc.collection, pl.targetId);
|
|
1174
|
+
const nTarget = SynapseConnectome.makeNodeId(nl.targetCollection || nextDoc.collection, nl.targetId);
|
|
1175
|
+
if (pTarget === nTarget) {
|
|
1176
|
+
return EMPTY_EDGE_DIFF;
|
|
1177
|
+
}
|
|
1178
|
+
return {
|
|
1179
|
+
edgesAdded: [nTarget],
|
|
1180
|
+
edgesRemoved: [pTarget]
|
|
1181
|
+
};
|
|
1182
|
+
}
|
|
1183
|
+
const prevTargets = new Set;
|
|
1184
|
+
const nextTargets = new Set;
|
|
1185
|
+
if (prevDoc) {
|
|
1186
|
+
for (let i = 0;i < prevLen; i++) {
|
|
1187
|
+
const link = prevDoc.wikilinks[i];
|
|
1188
|
+
prevTargets.add(SynapseConnectome.makeNodeId(link.targetCollection || prevDoc.collection, link.targetId));
|
|
1189
|
+
}
|
|
1190
|
+
}
|
|
1191
|
+
if (nextDoc) {
|
|
1192
|
+
for (let i = 0;i < nextLen; i++) {
|
|
1193
|
+
const link = nextDoc.wikilinks[i];
|
|
1194
|
+
nextTargets.add(SynapseConnectome.makeNodeId(link.targetCollection || nextDoc.collection, link.targetId));
|
|
1195
|
+
}
|
|
1196
|
+
}
|
|
1197
|
+
const edgesAdded = [];
|
|
1198
|
+
const edgesRemoved = [];
|
|
1199
|
+
for (const target of nextTargets) {
|
|
1200
|
+
if (!prevTargets.has(target)) {
|
|
1201
|
+
edgesAdded.push(target);
|
|
1202
|
+
}
|
|
1203
|
+
}
|
|
1204
|
+
for (const target of prevTargets) {
|
|
1205
|
+
if (!nextTargets.has(target)) {
|
|
1206
|
+
edgesRemoved.push(target);
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
return { edgesAdded, edgesRemoved };
|
|
1210
|
+
}
|
|
1211
|
+
recordDocumentUpsert(prevDoc, nextDoc, rawMarkdown = null, targetDiskPath = null, timestampMs) {
|
|
1212
|
+
const { edgesAdded, edgesRemoved } = CausalWalEngine.computeEdgeDiff(prevDoc, nextDoc);
|
|
1213
|
+
if (this.arena.isScopeActive()) {
|
|
1214
|
+
this.arena.allocSlot("DOC_UPSERT" /* DOC_UPSERT */, nextDoc.collection, nextDoc.id, prevDoc, nextDoc, edgesAdded, edgesRemoved, rawMarkdown, targetDiskPath);
|
|
1215
|
+
return null;
|
|
1216
|
+
}
|
|
1217
|
+
const entry = this.ringBuffer.append({
|
|
1218
|
+
timestampMs,
|
|
1219
|
+
eventType: "DOC_UPSERT" /* DOC_UPSERT */,
|
|
1220
|
+
collection: nextDoc.collection,
|
|
1221
|
+
docId: nextDoc.id,
|
|
1222
|
+
prevDoc,
|
|
1223
|
+
nextDoc,
|
|
1224
|
+
edgesAdded,
|
|
1225
|
+
edgesRemoved
|
|
1226
|
+
});
|
|
1227
|
+
if (this.sqliteStore) {
|
|
1228
|
+
this.sqliteStore.flushEntries([entry]);
|
|
1229
|
+
}
|
|
1230
|
+
return entry;
|
|
1231
|
+
}
|
|
1232
|
+
recordDocumentDelete(prevDoc, targetDiskPath = null, timestampMs) {
|
|
1233
|
+
const { edgesAdded, edgesRemoved } = CausalWalEngine.computeEdgeDiff(prevDoc, null);
|
|
1234
|
+
if (this.arena.isScopeActive()) {
|
|
1235
|
+
this.arena.allocSlot("DOC_DELETE" /* DOC_DELETE */, prevDoc.collection, prevDoc.id, prevDoc, null, edgesAdded, edgesRemoved, null, targetDiskPath);
|
|
1236
|
+
return null;
|
|
1237
|
+
}
|
|
1238
|
+
const entry = this.ringBuffer.append({
|
|
1239
|
+
timestampMs,
|
|
1240
|
+
eventType: "DOC_DELETE" /* DOC_DELETE */,
|
|
1241
|
+
collection: prevDoc.collection,
|
|
1242
|
+
docId: prevDoc.id,
|
|
1243
|
+
prevDoc,
|
|
1244
|
+
nextDoc: null,
|
|
1245
|
+
edgesAdded,
|
|
1246
|
+
edgesRemoved
|
|
1247
|
+
});
|
|
1248
|
+
if (this.sqliteStore) {
|
|
1249
|
+
this.sqliteStore.flushEntries([entry]);
|
|
1250
|
+
}
|
|
1251
|
+
return entry;
|
|
1252
|
+
}
|
|
1253
|
+
recordCompilerMetric(metric, timestampMs) {
|
|
1254
|
+
const entry = this.ringBuffer.append({
|
|
1255
|
+
timestampMs,
|
|
1256
|
+
eventType: "COMPILER_METRIC" /* COMPILER_METRIC */,
|
|
1257
|
+
collection: "_compiler_metrics",
|
|
1258
|
+
docId: metric.targetFile,
|
|
1259
|
+
prevDoc: null,
|
|
1260
|
+
nextDoc: null,
|
|
1261
|
+
edgesAdded: [],
|
|
1262
|
+
edgesRemoved: [],
|
|
1263
|
+
compilerMetric: metric
|
|
1264
|
+
});
|
|
1265
|
+
if (this.sqliteStore) {
|
|
1266
|
+
this.sqliteStore.flushEntries([entry]);
|
|
1267
|
+
}
|
|
1268
|
+
return entry;
|
|
1269
|
+
}
|
|
1270
|
+
executeArenaTransaction(fn, onCommitSlot) {
|
|
1271
|
+
this.arena.beginScope();
|
|
1272
|
+
try {
|
|
1273
|
+
const result = fn();
|
|
1274
|
+
const activeSlots = this.arena.getActiveSlots();
|
|
1275
|
+
const committedEntries = [];
|
|
1276
|
+
for (const slot of activeSlots) {
|
|
1277
|
+
onCommitSlot(slot);
|
|
1278
|
+
const entry = this.ringBuffer.append({
|
|
1279
|
+
eventType: slot.eventType,
|
|
1280
|
+
collection: slot.collection,
|
|
1281
|
+
docId: slot.docId,
|
|
1282
|
+
prevDoc: slot.prevDoc,
|
|
1283
|
+
nextDoc: slot.nextDoc,
|
|
1284
|
+
edgesAdded: [...slot.edgesAdded],
|
|
1285
|
+
edgesRemoved: [...slot.edgesRemoved],
|
|
1286
|
+
compilerMetric: slot.compilerMetric
|
|
1287
|
+
});
|
|
1288
|
+
committedEntries.push(entry);
|
|
1289
|
+
}
|
|
1290
|
+
this.arena.commitAndReset();
|
|
1291
|
+
if (this.sqliteStore && committedEntries.length > 0) {
|
|
1292
|
+
this.sqliteStore.flushEntries(committedEntries);
|
|
1293
|
+
}
|
|
1294
|
+
return result;
|
|
1295
|
+
} catch (err) {
|
|
1296
|
+
this.arena.rollbackAndReset();
|
|
1297
|
+
throw err;
|
|
1298
|
+
}
|
|
1299
|
+
}
|
|
1300
|
+
async executeArenaTransactionAsync(fn, onCommitSlot) {
|
|
1301
|
+
this.arena.beginScope();
|
|
1302
|
+
try {
|
|
1303
|
+
const result = await fn();
|
|
1304
|
+
const activeSlots = this.arena.getActiveSlots();
|
|
1305
|
+
const committedEntries = [];
|
|
1306
|
+
for (const slot of activeSlots) {
|
|
1307
|
+
onCommitSlot(slot);
|
|
1308
|
+
const entry = this.ringBuffer.append({
|
|
1309
|
+
eventType: slot.eventType,
|
|
1310
|
+
collection: slot.collection,
|
|
1311
|
+
docId: slot.docId,
|
|
1312
|
+
prevDoc: slot.prevDoc,
|
|
1313
|
+
nextDoc: slot.nextDoc,
|
|
1314
|
+
edgesAdded: [...slot.edgesAdded],
|
|
1315
|
+
edgesRemoved: [...slot.edgesRemoved],
|
|
1316
|
+
compilerMetric: slot.compilerMetric
|
|
1317
|
+
});
|
|
1318
|
+
committedEntries.push(entry);
|
|
1319
|
+
}
|
|
1320
|
+
this.arena.commitAndReset();
|
|
1321
|
+
if (this.sqliteStore && committedEntries.length > 0) {
|
|
1322
|
+
this.sqliteStore.flushEntries(committedEntries);
|
|
1323
|
+
}
|
|
1324
|
+
return result;
|
|
1325
|
+
} catch (err) {
|
|
1326
|
+
this.arena.rollbackAndReset();
|
|
1327
|
+
throw err;
|
|
1328
|
+
}
|
|
1329
|
+
}
|
|
1330
|
+
asOfLsn(connectome, targetLsn) {
|
|
1331
|
+
return CausalTimeTravelEngine.asOfLsn(connectome, this.ringBuffer, targetLsn);
|
|
1332
|
+
}
|
|
1333
|
+
asOf(connectome, timestampMsOrIso) {
|
|
1334
|
+
return CausalTimeTravelEngine.asOf(connectome, this.ringBuffer, timestampMsOrIso);
|
|
1335
|
+
}
|
|
1336
|
+
diffBetweenLsn(fromLsn, toLsn) {
|
|
1337
|
+
return CausalTimeTravelEngine.diffBetweenLsn(this.ringBuffer, fromLsn, toLsn);
|
|
1338
|
+
}
|
|
1339
|
+
getStats() {
|
|
1340
|
+
return {
|
|
1341
|
+
ringBuffer: this.ringBuffer.getStats(),
|
|
1342
|
+
arena: this.arena.getStats(),
|
|
1343
|
+
sqlite: {
|
|
1344
|
+
enabled: this.sqliteStore !== null,
|
|
1345
|
+
dbPath: this.sqliteStore?.dbPath ?? null,
|
|
1346
|
+
journalMode: this.sqliteStore ? this.sqliteStore.getJournalMode() : "disabled",
|
|
1347
|
+
lastFlushedLsn: this.sqliteStore ? this.sqliteStore.getLastFlushedLsn() : 0,
|
|
1348
|
+
persistedCount: this.sqliteStore ? this.sqliteStore.getPersistedCount() : 0
|
|
1349
|
+
}
|
|
1350
|
+
};
|
|
1351
|
+
}
|
|
1352
|
+
close() {
|
|
1353
|
+
if (this.sqliteStore) {
|
|
1354
|
+
this.sqliteStore.close();
|
|
1355
|
+
}
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1358
|
+
|
|
508
1359
|
// engine.ts
|
|
509
1360
|
class SynapseEngine {
|
|
510
1361
|
vaultDir;
|
|
511
1362
|
connectome;
|
|
1363
|
+
wal;
|
|
512
1364
|
config;
|
|
513
1365
|
isInitialized = false;
|
|
514
1366
|
pendingWriteQueue = new Map;
|
|
515
1367
|
writeTimer = null;
|
|
1368
|
+
nextTimerFlushAtMs = 0;
|
|
1369
|
+
cachedIsoMs = -1;
|
|
1370
|
+
cachedIsoStr = "";
|
|
516
1371
|
watcherDebounceTimer = null;
|
|
517
1372
|
internalWriteResetTimer = null;
|
|
518
1373
|
fileWatcher = null;
|
|
@@ -522,18 +1377,20 @@ class SynapseEngine {
|
|
|
522
1377
|
autoWatch: true,
|
|
523
1378
|
persistDebounceMs: 100,
|
|
524
1379
|
...config,
|
|
525
|
-
vaultDir:
|
|
1380
|
+
vaultDir: resolve2(config.vaultDir)
|
|
526
1381
|
};
|
|
527
1382
|
this.vaultDir = this.config.vaultDir;
|
|
528
1383
|
this.connectome = new SynapseConnectome;
|
|
1384
|
+
this.wal = new CausalWalEngine(this.vaultDir, this.config.wal);
|
|
529
1385
|
}
|
|
530
1386
|
async init() {
|
|
531
1387
|
if (this.isInitialized)
|
|
532
1388
|
return;
|
|
533
|
-
if (!
|
|
534
|
-
|
|
1389
|
+
if (!existsSync3(this.vaultDir)) {
|
|
1390
|
+
mkdirSync3(this.vaultDir, { recursive: true });
|
|
535
1391
|
}
|
|
536
1392
|
this.reloadVault();
|
|
1393
|
+
this.wal.initAndRecover(this.connectome);
|
|
537
1394
|
if (this.config.autoWatch) {
|
|
538
1395
|
this.startWatcher();
|
|
539
1396
|
}
|
|
@@ -541,13 +1398,13 @@ class SynapseEngine {
|
|
|
541
1398
|
}
|
|
542
1399
|
reloadVault() {
|
|
543
1400
|
this.connectome.clear();
|
|
544
|
-
if (!
|
|
1401
|
+
if (!existsSync3(this.vaultDir))
|
|
545
1402
|
return;
|
|
546
1403
|
const entries = readdirSync(this.vaultDir, { withFileTypes: true });
|
|
547
1404
|
for (const entry of entries) {
|
|
548
|
-
if (entry.isDirectory()) {
|
|
1405
|
+
if (entry.isDirectory() && !entry.name.startsWith(".")) {
|
|
549
1406
|
const collectionName = entry.name;
|
|
550
|
-
const colPath =
|
|
1407
|
+
const colPath = join2(this.vaultDir, collectionName);
|
|
551
1408
|
this.loadCollectionDir(collectionName, colPath);
|
|
552
1409
|
}
|
|
553
1410
|
}
|
|
@@ -559,7 +1416,7 @@ class SynapseEngine {
|
|
|
559
1416
|
if (!file.endsWith(".md") || file.includes(".tmp"))
|
|
560
1417
|
continue;
|
|
561
1418
|
const id = basename(file, ".md");
|
|
562
|
-
const fullPath =
|
|
1419
|
+
const fullPath = join2(dirPath, file);
|
|
563
1420
|
try {
|
|
564
1421
|
const raw = readFileSync2(fullPath, "utf-8");
|
|
565
1422
|
const parsed = parseMarkdownFile(raw, collectionName);
|
|
@@ -585,39 +1442,74 @@ class SynapseEngine {
|
|
|
585
1442
|
}
|
|
586
1443
|
} catch {}
|
|
587
1444
|
}
|
|
1445
|
+
getEffectiveDocument(collection, id) {
|
|
1446
|
+
if (this.wal.arena.isScopeActive()) {
|
|
1447
|
+
const slots = this.wal.arena.getActiveSlots();
|
|
1448
|
+
for (let i = slots.length - 1;i >= 0; i--) {
|
|
1449
|
+
const slot = slots[i];
|
|
1450
|
+
if (slot.collection === collection && slot.docId === id) {
|
|
1451
|
+
if (slot.eventType === "DOC_DELETE" /* DOC_DELETE */)
|
|
1452
|
+
return null;
|
|
1453
|
+
if (slot.eventType === "DOC_UPSERT" /* DOC_UPSERT */) {
|
|
1454
|
+
return slot.nextDoc || null;
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
}
|
|
1458
|
+
}
|
|
1459
|
+
return this.connectome.getDocument(collection, id);
|
|
1460
|
+
}
|
|
1461
|
+
isValidSafeIdentifier(str) {
|
|
1462
|
+
for (let i = 0;i < str.length; i++) {
|
|
1463
|
+
const code = str.charCodeAt(i);
|
|
1464
|
+
if (code === 47 || code === 92 || code === 58 || code === 42 || code === 63 || code === 34 || code === 60 || code === 62 || code === 124) {
|
|
1465
|
+
return false;
|
|
1466
|
+
}
|
|
1467
|
+
}
|
|
1468
|
+
return true;
|
|
1469
|
+
}
|
|
1470
|
+
getNowIso(nowMs) {
|
|
1471
|
+
if (nowMs === this.cachedIsoMs) {
|
|
1472
|
+
return this.cachedIsoStr;
|
|
1473
|
+
}
|
|
1474
|
+
this.cachedIsoMs = nowMs;
|
|
1475
|
+
this.cachedIsoStr = new Date(nowMs).toISOString();
|
|
1476
|
+
return this.cachedIsoStr;
|
|
1477
|
+
}
|
|
588
1478
|
async saveDocument(collection, input) {
|
|
589
|
-
const
|
|
590
|
-
const
|
|
1479
|
+
const nowMs = Date.now();
|
|
1480
|
+
const now = this.getNowIso(nowMs);
|
|
1481
|
+
const existing = this.getEffectiveDocument(collection, input.id);
|
|
591
1482
|
const createdAt = input.createdAt || existing?.createdAt || now;
|
|
592
1483
|
const updatedAt = input.updatedAt || now;
|
|
593
|
-
const
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
_createdAt: createdAt,
|
|
598
|
-
_updatedAt: updatedAt
|
|
599
|
-
}, cleanBody);
|
|
600
|
-
const parsed = parseMarkdownFile(rawMarkdown, collection);
|
|
601
|
-
const restoredFm = { ...parsed.frontmatter };
|
|
602
|
-
delete restoredFm._createdAt;
|
|
603
|
-
delete restoredFm._updatedAt;
|
|
1484
|
+
const rawBody = input.body || "";
|
|
1485
|
+
const cleanBody = rawBody.length > 0 && (rawBody.includes("\r") || rawBody.charCodeAt(0) === 10 || rawBody.charCodeAt(rawBody.length - 1) === 10) ? rawBody.replace(/\r\n/g, `
|
|
1486
|
+
`).replace(/^\n+/, "").replace(/\n+$/, "") : rawBody;
|
|
1487
|
+
const wikilinks = extractWikilinksFromDataAndBody(input.data, cleanBody, collection);
|
|
604
1488
|
const finalDoc = {
|
|
605
1489
|
id: input.id,
|
|
606
1490
|
collection,
|
|
607
|
-
data:
|
|
608
|
-
body:
|
|
1491
|
+
data: input.data,
|
|
1492
|
+
body: cleanBody,
|
|
609
1493
|
createdAt,
|
|
610
1494
|
updatedAt,
|
|
611
|
-
wikilinks
|
|
1495
|
+
wikilinks
|
|
612
1496
|
};
|
|
1497
|
+
const safeCol = this.isValidSafeIdentifier(collection) ? collection : collection.replace(/[\/\\:*?"<>|]/g, "_");
|
|
1498
|
+
const safeId = this.isValidSafeIdentifier(input.id) ? input.id : input.id.replace(/[\/\\:*?"<>|]/g, "_");
|
|
1499
|
+
const targetPath = `${this.vaultDir}/${safeCol}/${safeId}.md`;
|
|
1500
|
+
this.wal.recordDocumentUpsert(existing, finalDoc, null, targetPath, nowMs);
|
|
1501
|
+
if (this.wal.arena.isScopeActive()) {
|
|
1502
|
+
return finalDoc;
|
|
1503
|
+
}
|
|
613
1504
|
this.connectome.putDocument(finalDoc);
|
|
614
|
-
const safeCol = collection.replace(/[\/\\:*?"<>|]/g, "_");
|
|
615
|
-
const safeId = input.id.replace(/[\/\\:*?"<>|]/g, "_");
|
|
616
|
-
const targetDir = join(this.vaultDir, safeCol);
|
|
617
|
-
const targetPath = join(targetDir, `${safeId}.md`);
|
|
618
1505
|
const writeKey = `${collection}/${input.id}`;
|
|
619
|
-
this.scheduleWrite(writeKey, () => {
|
|
1506
|
+
this.scheduleWrite(writeKey, nowMs, () => {
|
|
620
1507
|
this.markInternalWrite();
|
|
1508
|
+
const rawMarkdown = stringifyDocument({
|
|
1509
|
+
...finalDoc.data,
|
|
1510
|
+
_createdAt: finalDoc.createdAt,
|
|
1511
|
+
_updatedAt: finalDoc.updatedAt
|
|
1512
|
+
}, finalDoc.body);
|
|
621
1513
|
atomicWriteFile(targetPath, rawMarkdown);
|
|
622
1514
|
});
|
|
623
1515
|
return finalDoc;
|
|
@@ -633,30 +1525,79 @@ class SynapseEngine {
|
|
|
633
1525
|
}, 120);
|
|
634
1526
|
}
|
|
635
1527
|
async deleteDocument(collection, id) {
|
|
636
|
-
const
|
|
637
|
-
if (!
|
|
1528
|
+
const existing = this.getEffectiveDocument(collection, id);
|
|
1529
|
+
if (!existing)
|
|
638
1530
|
return false;
|
|
639
1531
|
const safeCol = collection.replace(/[\/\\:*?"<>|]/g, "_");
|
|
640
1532
|
const safeId = id.replace(/[\/\\:*?"<>|]/g, "_");
|
|
641
|
-
const targetPath =
|
|
1533
|
+
const targetPath = join2(this.vaultDir, safeCol, `${safeId}.md`);
|
|
1534
|
+
this.wal.recordDocumentDelete(existing, targetPath);
|
|
1535
|
+
if (this.wal.arena.isScopeActive()) {
|
|
1536
|
+
return true;
|
|
1537
|
+
}
|
|
1538
|
+
const deleted = this.connectome.removeDocument(collection, id);
|
|
1539
|
+
if (!deleted)
|
|
1540
|
+
return false;
|
|
642
1541
|
const writeKey = `${collection}/${id}`;
|
|
643
1542
|
this.pendingWriteQueue.delete(writeKey);
|
|
644
1543
|
try {
|
|
645
|
-
if (
|
|
1544
|
+
if (existsSync3(targetPath)) {
|
|
646
1545
|
this.markInternalWrite();
|
|
647
1546
|
unlinkSync2(targetPath);
|
|
648
1547
|
}
|
|
649
1548
|
} catch {}
|
|
650
1549
|
return true;
|
|
651
1550
|
}
|
|
652
|
-
|
|
1551
|
+
async executeTransaction(fn) {
|
|
1552
|
+
return this.wal.executeArenaTransactionAsync(fn, (slot) => {
|
|
1553
|
+
if (slot.eventType === "DOC_UPSERT" /* DOC_UPSERT */ && slot.nextDoc) {
|
|
1554
|
+
const committedDoc = slot.nextDoc;
|
|
1555
|
+
this.connectome.putDocument(committedDoc);
|
|
1556
|
+
if (slot.targetDiskPath) {
|
|
1557
|
+
const targetPath = slot.targetDiskPath;
|
|
1558
|
+
const writeKey = `${slot.collection}/${slot.docId}`;
|
|
1559
|
+
this.scheduleWrite(writeKey, Date.now(), () => {
|
|
1560
|
+
this.markInternalWrite();
|
|
1561
|
+
const rawMarkdown = slot.rawMarkdown ?? stringifyDocument({
|
|
1562
|
+
...committedDoc.data,
|
|
1563
|
+
_createdAt: committedDoc.createdAt,
|
|
1564
|
+
_updatedAt: committedDoc.updatedAt
|
|
1565
|
+
}, committedDoc.body);
|
|
1566
|
+
atomicWriteFile(targetPath, rawMarkdown);
|
|
1567
|
+
});
|
|
1568
|
+
}
|
|
1569
|
+
} else if (slot.eventType === "DOC_DELETE" /* DOC_DELETE */) {
|
|
1570
|
+
this.connectome.removeDocument(slot.collection, slot.docId);
|
|
1571
|
+
const writeKey = `${slot.collection}/${slot.docId}`;
|
|
1572
|
+
this.pendingWriteQueue.delete(writeKey);
|
|
1573
|
+
if (slot.targetDiskPath) {
|
|
1574
|
+
try {
|
|
1575
|
+
if (existsSync3(slot.targetDiskPath)) {
|
|
1576
|
+
this.markInternalWrite();
|
|
1577
|
+
unlinkSync2(slot.targetDiskPath);
|
|
1578
|
+
}
|
|
1579
|
+
} catch {}
|
|
1580
|
+
}
|
|
1581
|
+
}
|
|
1582
|
+
});
|
|
1583
|
+
}
|
|
1584
|
+
scheduleWrite(key, nowMs, fn) {
|
|
653
1585
|
this.pendingWriteQueue.set(key, fn);
|
|
654
|
-
if (this.writeTimer)
|
|
655
|
-
clearTimeout(this.writeTimer);
|
|
656
1586
|
const debounceMs = this.config.persistDebounceMs || 100;
|
|
1587
|
+
this.nextTimerFlushAtMs = nowMs + debounceMs;
|
|
1588
|
+
if (!this.writeTimer) {
|
|
1589
|
+
this.armWriteTimer(debounceMs);
|
|
1590
|
+
}
|
|
1591
|
+
}
|
|
1592
|
+
armWriteTimer(delayMs) {
|
|
657
1593
|
this.writeTimer = setTimeout(() => {
|
|
1594
|
+
const remaining = this.nextTimerFlushAtMs - Date.now();
|
|
1595
|
+
if (remaining > 5 && this.pendingWriteQueue.size > 0) {
|
|
1596
|
+
this.armWriteTimer(remaining);
|
|
1597
|
+
return;
|
|
1598
|
+
}
|
|
658
1599
|
this.flushPendingWrites();
|
|
659
|
-
},
|
|
1600
|
+
}, delayMs);
|
|
660
1601
|
}
|
|
661
1602
|
flushPendingWrites() {
|
|
662
1603
|
if (this.writeTimer) {
|
|
@@ -715,6 +1656,7 @@ class SynapseEngine {
|
|
|
715
1656
|
this.fileWatcher.close();
|
|
716
1657
|
this.fileWatcher = null;
|
|
717
1658
|
}
|
|
1659
|
+
this.wal.close();
|
|
718
1660
|
}
|
|
719
1661
|
}
|
|
720
1662
|
|
|
@@ -765,7 +1707,7 @@ class SynapseCollection {
|
|
|
765
1707
|
}
|
|
766
1708
|
|
|
767
1709
|
// migrator.ts
|
|
768
|
-
import { existsSync as
|
|
1710
|
+
import { existsSync as existsSync4 } from "fs";
|
|
769
1711
|
|
|
770
1712
|
class SynapseMigrator {
|
|
771
1713
|
engine;
|
|
@@ -773,7 +1715,7 @@ class SynapseMigrator {
|
|
|
773
1715
|
this.engine = engine;
|
|
774
1716
|
}
|
|
775
1717
|
async migrateFromSqlite(sqliteDbPath, mappings) {
|
|
776
|
-
if (!
|
|
1718
|
+
if (!existsSync4(sqliteDbPath)) {
|
|
777
1719
|
throw new Error(`[SynapseMigrator] SQLite DB \uD30C\uC77C\uC744 \uCC3E\uC744 \uC218 \uC5C6\uC2B5\uB2C8\uB2E4: ${sqliteDbPath}`);
|
|
778
1720
|
}
|
|
779
1721
|
for (const m of mappings) {
|
|
@@ -923,19 +1865,45 @@ class SynapseDB {
|
|
|
923
1865
|
flush() {
|
|
924
1866
|
this.engine.flushPendingWrites();
|
|
925
1867
|
}
|
|
1868
|
+
async transaction(fn) {
|
|
1869
|
+
return this.engine.executeTransaction(fn);
|
|
1870
|
+
}
|
|
1871
|
+
asOfLsn(targetLsn) {
|
|
1872
|
+
return this.engine.wal.asOfLsn(this.engine.connectome, targetLsn);
|
|
1873
|
+
}
|
|
1874
|
+
asOf(timestampMsOrIso) {
|
|
1875
|
+
return this.engine.wal.asOf(this.engine.connectome, timestampMsOrIso);
|
|
1876
|
+
}
|
|
1877
|
+
diffBetweenLsn(fromLsn, toLsn) {
|
|
1878
|
+
return this.engine.wal.diffBetweenLsn(fromLsn, toLsn);
|
|
1879
|
+
}
|
|
1880
|
+
recordCompilerMetric(metric) {
|
|
1881
|
+
return this.engine.wal.recordCompilerMetric(metric);
|
|
1882
|
+
}
|
|
1883
|
+
walStats() {
|
|
1884
|
+
return this.engine.wal.getStats();
|
|
1885
|
+
}
|
|
926
1886
|
close() {
|
|
927
1887
|
this.engine.close();
|
|
928
1888
|
}
|
|
929
1889
|
}
|
|
930
1890
|
export {
|
|
1891
|
+
CausalArenaAllocator,
|
|
1892
|
+
CausalTimeTravelEngine,
|
|
1893
|
+
CausalWalEngine,
|
|
1894
|
+
CausalWalEventType,
|
|
1895
|
+
SqliteCausalWalStore,
|
|
931
1896
|
SynapseCollection,
|
|
932
1897
|
SynapseConnectome,
|
|
933
1898
|
SynapseDB,
|
|
934
1899
|
SynapseEngine,
|
|
935
1900
|
SynapseMigrator,
|
|
1901
|
+
TemporalRingBuffer,
|
|
1902
|
+
WAL_CONFIG_DEFAULTS,
|
|
936
1903
|
atomicWriteFile,
|
|
937
1904
|
exportToObsidianCanvas,
|
|
938
1905
|
extractWikilinks,
|
|
1906
|
+
extractWikilinksFromDataAndBody,
|
|
939
1907
|
parseMarkdownFile,
|
|
940
1908
|
parseYamlScalar,
|
|
941
1909
|
stringifyDocument
|