sigma-memory 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +11 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +36 -33
- package/dist/index.js.map +1 -1
- package/dist/types.d.ts +11 -10
- package/dist/types.d.ts.map +1 -1
- package/dist/vector-store.d.ts +100 -0
- package/dist/vector-store.d.ts.map +1 -0
- package/dist/vector-store.js +322 -0
- package/dist/vector-store.js.map +1 -0
- package/package.json +10 -6
- package/src/index.ts +38 -32
- package/src/types.ts +14 -12
- package/src/vector-store.ts +384 -0
- package/src/vendor.d.ts +30 -0
- package/dist/qmd.d.ts +0 -35
- package/dist/qmd.d.ts.map +0 -1
- package/dist/qmd.js +0 -162
- package/dist/qmd.js.map +0 -1
- package/src/qmd.ts +0 -180
package/dist/index.d.ts
CHANGED
|
@@ -1,21 +1,27 @@
|
|
|
1
1
|
import { NotesManager } from './notes.js';
|
|
2
2
|
import { OntologyManager } from './ontology.js';
|
|
3
|
-
import {
|
|
3
|
+
import { VectorStore } from './vector-store.js';
|
|
4
4
|
import type { MemoryConfig, UnifiedSearchResult, MemoryStatus } from './types.js';
|
|
5
5
|
export declare class SigmaMemory {
|
|
6
6
|
readonly notes: NotesManager;
|
|
7
7
|
readonly ontology: OntologyManager;
|
|
8
|
-
readonly
|
|
8
|
+
readonly vectors: VectorStore;
|
|
9
9
|
private readonly config;
|
|
10
10
|
constructor(config?: Partial<MemoryConfig>);
|
|
11
11
|
/**
|
|
12
|
-
* Unified search: searches notes + ontology +
|
|
12
|
+
* Unified search: searches notes + ontology + vectors, combines results
|
|
13
13
|
*/
|
|
14
14
|
search(query: string): Promise<UnifiedSearchResult[]>;
|
|
15
15
|
/**
|
|
16
|
-
* Initialize all required directories
|
|
16
|
+
* Initialize all required directories and the vector store.
|
|
17
|
+
* On first run, this will download the embedding model and index notes.
|
|
17
18
|
*/
|
|
18
19
|
init(): Promise<void>;
|
|
20
|
+
/**
|
|
21
|
+
* Index all markdown notes into the vector store.
|
|
22
|
+
* Reads every .md file from the notes directory and adds it.
|
|
23
|
+
*/
|
|
24
|
+
indexNotes(): Promise<void>;
|
|
19
25
|
/**
|
|
20
26
|
* Status of all subsystems
|
|
21
27
|
*/
|
|
@@ -27,7 +33,7 @@ export declare class SigmaMemory {
|
|
|
27
33
|
}
|
|
28
34
|
export { NotesManager } from './notes.js';
|
|
29
35
|
export { OntologyManager } from './ontology.js';
|
|
30
|
-
export {
|
|
36
|
+
export { VectorStore } from './vector-store.js';
|
|
31
37
|
export * from './types.js';
|
|
32
38
|
export default SigmaMemory;
|
|
33
39
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,KAAK,EAAE,YAAY,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAElF,qBAAa,WAAW;IACtB,SAAgB,KAAK,EAAE,YAAY,CAAC;IACpC,SAAgB,QAAQ,EAAE,eAAe,CAAC;IAC1C,SAAgB,OAAO,EAAE,WAAW,CAAC;IACrC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;gBAE1B,MAAM,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC;IAgB1C;;OAEG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,EAAE,CAAC;IAiE3D;;;OAGG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAiB3B;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAajC;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,YAAY,CAAC;IA6BrC;;OAEG;IACH,SAAS,IAAI,YAAY;CAG1B;AAGD,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,cAAc,YAAY,CAAC;AAG3B,eAAe,WAAW,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -14,46 +14,44 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.
|
|
17
|
+
exports.VectorStore = exports.OntologyManager = exports.NotesManager = exports.SigmaMemory = void 0;
|
|
18
18
|
const path_1 = require("path");
|
|
19
19
|
const os_1 = require("os");
|
|
20
20
|
const fs_1 = require("fs");
|
|
21
21
|
const notes_js_1 = require("./notes.js");
|
|
22
22
|
const ontology_js_1 = require("./ontology.js");
|
|
23
|
-
const
|
|
23
|
+
const vector_store_js_1 = require("./vector-store.js");
|
|
24
24
|
class SigmaMemory {
|
|
25
25
|
notes;
|
|
26
26
|
ontology;
|
|
27
|
-
|
|
27
|
+
vectors;
|
|
28
28
|
config;
|
|
29
29
|
constructor(config) {
|
|
30
30
|
// Default configuration
|
|
31
31
|
const defaultConfig = {
|
|
32
32
|
memoryDir: (0, path_1.join)((0, os_1.homedir)(), '.phi', 'memory'),
|
|
33
33
|
projectMemoryDir: (0, path_1.join)(process.cwd(), '.phi', 'memory'),
|
|
34
|
-
ontologyPath: (0, path_1.join)((0, os_1.homedir)(), '.phi', 'memory', 'ontology', 'graph.jsonl')
|
|
35
|
-
qmdEnabled: true,
|
|
36
|
-
qmdCommand: 'qmd'
|
|
34
|
+
ontologyPath: (0, path_1.join)((0, os_1.homedir)(), '.phi', 'memory', 'ontology', 'graph.jsonl')
|
|
37
35
|
};
|
|
38
36
|
this.config = { ...defaultConfig, ...config };
|
|
39
37
|
// Initialize managers
|
|
40
38
|
this.notes = new notes_js_1.NotesManager(this.config);
|
|
41
39
|
this.ontology = new ontology_js_1.OntologyManager(this.config);
|
|
42
|
-
this.
|
|
40
|
+
this.vectors = new vector_store_js_1.VectorStore((0, path_1.join)(this.config.memoryDir, 'vectors.db'));
|
|
43
41
|
}
|
|
44
42
|
/**
|
|
45
|
-
* Unified search: searches notes + ontology +
|
|
43
|
+
* Unified search: searches notes + ontology + vectors, combines results
|
|
46
44
|
*/
|
|
47
45
|
async search(query) {
|
|
48
46
|
const results = [];
|
|
49
|
-
// Search in notes
|
|
47
|
+
// Search in notes (full-text grep)
|
|
50
48
|
try {
|
|
51
49
|
const notesResults = this.notes.search(query);
|
|
52
50
|
for (const result of notesResults) {
|
|
53
51
|
results.push({
|
|
54
52
|
source: 'notes',
|
|
55
53
|
type: 'note',
|
|
56
|
-
score: 0.8, // Default score for notes
|
|
54
|
+
score: 0.8, // Default score for text-match notes
|
|
57
55
|
data: result
|
|
58
56
|
});
|
|
59
57
|
}
|
|
@@ -86,12 +84,12 @@ class SigmaMemory {
|
|
|
86
84
|
catch (error) {
|
|
87
85
|
// Ontology search failed silently
|
|
88
86
|
}
|
|
89
|
-
//
|
|
87
|
+
// Vector similarity search
|
|
90
88
|
try {
|
|
91
|
-
const
|
|
92
|
-
for (const result of
|
|
89
|
+
const vectorResults = await this.vectors.search(query, 5);
|
|
90
|
+
for (const result of vectorResults) {
|
|
93
91
|
results.push({
|
|
94
|
-
source: '
|
|
92
|
+
source: 'vectors',
|
|
95
93
|
type: 'file',
|
|
96
94
|
score: result.score,
|
|
97
95
|
data: result
|
|
@@ -99,14 +97,15 @@ class SigmaMemory {
|
|
|
99
97
|
}
|
|
100
98
|
}
|
|
101
99
|
catch (error) {
|
|
102
|
-
//
|
|
100
|
+
// Vector search failed silently
|
|
103
101
|
}
|
|
104
102
|
// Sort by score descending
|
|
105
103
|
results.sort((a, b) => b.score - a.score);
|
|
106
104
|
return results;
|
|
107
105
|
}
|
|
108
106
|
/**
|
|
109
|
-
* Initialize all required directories
|
|
107
|
+
* Initialize all required directories and the vector store.
|
|
108
|
+
* On first run, this will download the embedding model and index notes.
|
|
110
109
|
*/
|
|
111
110
|
async init() {
|
|
112
111
|
// Create base directories
|
|
@@ -116,13 +115,24 @@ class SigmaMemory {
|
|
|
116
115
|
if (!(0, fs_1.existsSync)(this.config.projectMemoryDir)) {
|
|
117
116
|
(0, fs_1.mkdirSync)(this.config.projectMemoryDir, { recursive: true });
|
|
118
117
|
}
|
|
119
|
-
// Initialize
|
|
120
|
-
|
|
118
|
+
// Initialize vector store (DB setup only — fast)
|
|
119
|
+
await this.vectors.init();
|
|
120
|
+
// Auto-index existing notes into the vector store
|
|
121
|
+
await this.indexNotes();
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Index all markdown notes into the vector store.
|
|
125
|
+
* Reads every .md file from the notes directory and adds it.
|
|
126
|
+
*/
|
|
127
|
+
async indexNotes() {
|
|
128
|
+
const notesList = this.notes.list();
|
|
129
|
+
for (const note of notesList) {
|
|
121
130
|
try {
|
|
122
|
-
|
|
131
|
+
const content = this.notes.read(note.name);
|
|
132
|
+
await this.vectors.addDocument(note.name, content);
|
|
123
133
|
}
|
|
124
|
-
catch
|
|
125
|
-
//
|
|
134
|
+
catch {
|
|
135
|
+
// Skip files that can't be read
|
|
126
136
|
}
|
|
127
137
|
}
|
|
128
138
|
}
|
|
@@ -146,19 +156,12 @@ class SigmaMemory {
|
|
|
146
156
|
entitiesByType: ontologyStats.entitiesByType,
|
|
147
157
|
relationsByType: ontologyStats.relationsByType
|
|
148
158
|
};
|
|
149
|
-
//
|
|
150
|
-
|
|
151
|
-
if (this.config.qmdEnabled && this.qmd.isAvailable()) {
|
|
152
|
-
const status = await this.qmd.status();
|
|
153
|
-
qmdStatus = {
|
|
154
|
-
available: true,
|
|
155
|
-
status: status || { files: 0, chunks: 0, lastUpdate: null }
|
|
156
|
-
};
|
|
157
|
-
}
|
|
159
|
+
// Vector store status
|
|
160
|
+
const vectorStats = this.vectors.getStats();
|
|
158
161
|
return {
|
|
159
162
|
notes: notesStatus,
|
|
160
163
|
ontology: ontologyStatus,
|
|
161
|
-
|
|
164
|
+
vectors: vectorStats
|
|
162
165
|
};
|
|
163
166
|
}
|
|
164
167
|
/**
|
|
@@ -174,8 +177,8 @@ var notes_js_2 = require("./notes.js");
|
|
|
174
177
|
Object.defineProperty(exports, "NotesManager", { enumerable: true, get: function () { return notes_js_2.NotesManager; } });
|
|
175
178
|
var ontology_js_2 = require("./ontology.js");
|
|
176
179
|
Object.defineProperty(exports, "OntologyManager", { enumerable: true, get: function () { return ontology_js_2.OntologyManager; } });
|
|
177
|
-
var
|
|
178
|
-
Object.defineProperty(exports, "
|
|
180
|
+
var vector_store_js_2 = require("./vector-store.js");
|
|
181
|
+
Object.defineProperty(exports, "VectorStore", { enumerable: true, get: function () { return vector_store_js_2.VectorStore; } });
|
|
179
182
|
__exportStar(require("./types.js"), exports);
|
|
180
183
|
// Default export
|
|
181
184
|
exports.default = SigmaMemory;
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAAA,+BAA4B;AAC5B,2BAA6B;AAC7B,2BAA2C;AAC3C,yCAA0C;AAC1C,+CAAgD;AAChD,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAAA,+BAA4B;AAC5B,2BAA6B;AAC7B,2BAA2C;AAC3C,yCAA0C;AAC1C,+CAAgD;AAChD,uDAAgD;AAGhD,MAAa,WAAW;IACN,KAAK,CAAe;IACpB,QAAQ,CAAkB;IAC1B,OAAO,CAAc;IACpB,MAAM,CAAe;IAEtC,YAAY,MAA8B;QACxC,wBAAwB;QACxB,MAAM,aAAa,GAAiB;YAClC,SAAS,EAAE,IAAA,WAAI,EAAC,IAAA,YAAO,GAAE,EAAE,MAAM,EAAE,QAAQ,CAAC;YAC5C,gBAAgB,EAAE,IAAA,WAAI,EAAC,OAAO,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,QAAQ,CAAC;YACvD,YAAY,EAAE,IAAA,WAAI,EAAC,IAAA,YAAO,GAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,aAAa,CAAC;SAC3E,CAAC;QAEF,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,aAAa,EAAE,GAAG,MAAM,EAAE,CAAC;QAE9C,sBAAsB;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,uBAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,CAAC,QAAQ,GAAG,IAAI,6BAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACjD,IAAI,CAAC,OAAO,GAAG,IAAI,6BAAW,CAAC,IAAA,WAAI,EAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,KAAa;QACxB,MAAM,OAAO,GAA0B,EAAE,CAAC;QAE1C,mCAAmC;QACnC,IAAI,CAAC;YACH,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC9C,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;gBAClC,OAAO,CAAC,IAAI,CAAC;oBACX,MAAM,EAAE,OAAO;oBACf,IAAI,EAAE,MAAM;oBACZ,KAAK,EAAE,GAAG,EAAE,qCAAqC;oBACjD,IAAI,EAAE,MAAM;iBACb,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,+BAA+B;QACjC,CAAC;QAED,qBAAqB;QACrB,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YAChE,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;gBACnC,OAAO,CAAC,IAAI,CAAC;oBACX,MAAM,EAAE,UAAU;oBAClB,IAAI,EAAE,QAAQ;oBACd,KAAK,EAAE,GAAG;oBACV,IAAI,EAAE,MAAM;iBACb,CAAC,CAAC;gBAEH,oCAAoC;gBACpC,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACzD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;oBACjC,OAAO,CAAC,IAAI,CAAC;wBACX,MAAM,EAAE,UAAU;wBAClB,IAAI,EAAE,UAAU;wBAChB,KAAK,EAAE,GAAG;wBACV,IAAI,EAAE,QAAQ;qBACf,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,kCAAkC;QACpC,CAAC;QAED,2BAA2B;QAC3B,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YAC1D,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;gBACnC,OAAO,CAAC,IAAI,CAAC;oBACX,MAAM,EAAE,SAAS;oBACjB,IAAI,EAAE,MAAM;oBACZ,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,IAAI,EAAE,MAAM;iBACb,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,gCAAgC;QAClC,CAAC;QAED,2BAA2B;QAC3B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAE1C,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,IAAI;QACR,0BAA0B;QAC1B,IAAI,CAAC,IAAA,eAAU,EAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YACvC,IAAA,cAAS,EAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,IAAI,CAAC,IAAA,eAAU,EAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAC9C,IAAA,cAAS,EAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/D,CAAC;QAED,iDAAiD;QACjD,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;QAE1B,kDAAkD;QAClD,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;IAC1B,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACd,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAEpC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC7B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC3C,MAAM,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACrD,CAAC;YAAC,MAAM,CAAC;gBACP,gCAAgC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM;QACV,eAAe;QACf,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,WAAW,GAAG;YAClB,KAAK,EAAE,SAAS,CAAC,MAAM;YACvB,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YAC9D,YAAY,EAAE,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;SAC9D,CAAC;QAEF,kBAAkB;QAClB,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QAC5C,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC/C,MAAM,cAAc,GAAG;YACrB,QAAQ,EAAE,aAAa,CAAC,QAAQ,CAAC,MAAM;YACvC,SAAS,EAAE,aAAa,CAAC,SAAS,CAAC,MAAM;YACzC,cAAc,EAAE,aAAa,CAAC,cAAc;YAC5C,eAAe,EAAE,aAAa,CAAC,eAAe;SAC/C,CAAC;QAEF,sBAAsB;QACtB,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;QAE5C,OAAO;YACL,KAAK,EAAE,WAAW;YAClB,QAAQ,EAAE,cAAc;YACxB,OAAO,EAAE,WAAW;SACrB,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,SAAS;QACP,OAAO,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;IAC5B,CAAC;CACF;AAtKD,kCAsKC;AAED,qBAAqB;AACrB,uCAA0C;AAAjC,wGAAA,YAAY,OAAA;AACrB,6CAAgD;AAAvC,8GAAA,eAAe,OAAA;AACxB,qDAAgD;AAAvC,8GAAA,WAAW,OAAA;AACpB,6CAA2B;AAE3B,iBAAiB;AACjB,kBAAe,WAAW,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -2,8 +2,6 @@ export interface MemoryConfig {
|
|
|
2
2
|
memoryDir: string;
|
|
3
3
|
projectMemoryDir: string;
|
|
4
4
|
ontologyPath: string;
|
|
5
|
-
qmdEnabled: boolean;
|
|
6
|
-
qmdCommand?: string;
|
|
7
5
|
}
|
|
8
6
|
export interface SearchResult {
|
|
9
7
|
file: string;
|
|
@@ -11,6 +9,12 @@ export interface SearchResult {
|
|
|
11
9
|
content: string;
|
|
12
10
|
score: number;
|
|
13
11
|
}
|
|
12
|
+
export interface VectorSearchResult {
|
|
13
|
+
file: string;
|
|
14
|
+
chunkIndex: number;
|
|
15
|
+
content: string;
|
|
16
|
+
score: number;
|
|
17
|
+
}
|
|
14
18
|
export interface OntologyEntity {
|
|
15
19
|
id: string;
|
|
16
20
|
type: 'Person' | 'Project' | 'Device' | 'Account' | 'Document' | 'Service' | 'Concept';
|
|
@@ -33,7 +37,7 @@ export interface Note {
|
|
|
33
37
|
content: string;
|
|
34
38
|
}
|
|
35
39
|
export interface UnifiedSearchResult {
|
|
36
|
-
source: 'notes' | 'ontology' | '
|
|
40
|
+
source: 'notes' | 'ontology' | 'vectors';
|
|
37
41
|
type?: 'entity' | 'relation' | 'note' | 'file';
|
|
38
42
|
score: number;
|
|
39
43
|
data: any;
|
|
@@ -50,13 +54,10 @@ export interface MemoryStatus {
|
|
|
50
54
|
entitiesByType: Record<string, number>;
|
|
51
55
|
relationsByType: Record<string, number>;
|
|
52
56
|
};
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
chunks: number;
|
|
58
|
-
lastUpdate: string | null;
|
|
59
|
-
};
|
|
57
|
+
vectors: {
|
|
58
|
+
documentCount: number;
|
|
59
|
+
chunkCount: number;
|
|
60
|
+
lastUpdate: string;
|
|
60
61
|
};
|
|
61
62
|
}
|
|
62
63
|
export interface OntologyEntityEntry {
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,QAAQ,GAAG,SAAS,GAAG,QAAQ,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,GAAG,SAAS,CAAC;IACvF,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,IAAI;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,OAAO,GAAG,UAAU,GAAG,SAAS,CAAC;IACzC,IAAI,CAAC,EAAE,QAAQ,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,CAAC;IAC/C,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,GAAG,CAAC;CACX;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE;QACL,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;KAC7B,CAAC;IACF,QAAQ,EAAE;QACR,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACvC,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KACzC,CAAC;IACF,OAAO,EAAE;QACP,aAAa,EAAE,MAAM,CAAC;QACtB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;KACpB,CAAC;CACH;AAGD,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,QAAQ,CAAC;IACf,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,QAAQ,GAAG,SAAS,GAAG,QAAQ,GAAG,SAAS,GAAG,UAAU,GAAG,SAAS,GAAG,SAAS,CAAC;IACvF,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,UAAU,CAAC;IACjB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,QAAQ,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,kBAAkB,GAAG,mBAAmB,GAAG,qBAAqB,GAAG,mBAAmB,CAAC"}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import type { VectorSearchResult } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Self-contained vector store using sql.js (SQLite via WebAssembly) and
|
|
4
|
+
* @huggingface/transformers for local embeddings.
|
|
5
|
+
*
|
|
6
|
+
* Zero configuration — works out of the box on any platform.
|
|
7
|
+
* No native compilation required.
|
|
8
|
+
*/
|
|
9
|
+
export declare class VectorStore {
|
|
10
|
+
private db;
|
|
11
|
+
private pipeline;
|
|
12
|
+
private dbPath;
|
|
13
|
+
private initialized;
|
|
14
|
+
private initPromise;
|
|
15
|
+
private modelPromise;
|
|
16
|
+
constructor(dbPath: string);
|
|
17
|
+
/**
|
|
18
|
+
* Initialize the vector store: sets up the SQLite database.
|
|
19
|
+
* The embedding model is loaded lazily on first embed operation.
|
|
20
|
+
* Safe to call multiple times — only initializes once.
|
|
21
|
+
*/
|
|
22
|
+
init(): Promise<void>;
|
|
23
|
+
private _init;
|
|
24
|
+
/**
|
|
25
|
+
* Load the embedding model. Called lazily on first embed operation.
|
|
26
|
+
* Downloads the model on first use (~23MB for all-MiniLM-L6-v2).
|
|
27
|
+
*/
|
|
28
|
+
private loadEmbeddingModel;
|
|
29
|
+
/**
|
|
30
|
+
* Ensure the embedding model is loaded before use.
|
|
31
|
+
*/
|
|
32
|
+
private ensurePipeline;
|
|
33
|
+
/**
|
|
34
|
+
* Persist the in-memory SQLite database to disk.
|
|
35
|
+
*/
|
|
36
|
+
private persist;
|
|
37
|
+
/**
|
|
38
|
+
* Embed text into a Float32Array vector (384 dimensions).
|
|
39
|
+
*/
|
|
40
|
+
private embed;
|
|
41
|
+
/**
|
|
42
|
+
* Chunk text into overlapping segments.
|
|
43
|
+
*
|
|
44
|
+
* Strategy:
|
|
45
|
+
* 1. Split by paragraphs (double newline)
|
|
46
|
+
* 2. If a paragraph exceeds 500 chars, split by sentences
|
|
47
|
+
* 3. Each chunk gets a 100-char overlap with the previous chunk
|
|
48
|
+
*/
|
|
49
|
+
private chunkText;
|
|
50
|
+
/**
|
|
51
|
+
* Serialize a Float32Array to a Buffer for BLOB storage.
|
|
52
|
+
*/
|
|
53
|
+
private serializeEmbedding;
|
|
54
|
+
/**
|
|
55
|
+
* Deserialize a BLOB (Uint8Array) back to Float32Array.
|
|
56
|
+
*/
|
|
57
|
+
private deserializeEmbedding;
|
|
58
|
+
/**
|
|
59
|
+
* Compute cosine similarity between two vectors.
|
|
60
|
+
* Both vectors should be normalized (which they are from the model),
|
|
61
|
+
* so this is equivalent to the dot product.
|
|
62
|
+
*/
|
|
63
|
+
private cosineSimilarity;
|
|
64
|
+
/**
|
|
65
|
+
* Add a document to the vector store.
|
|
66
|
+
* Chunks the content, embeds each chunk, and stores in the DB.
|
|
67
|
+
* Replaces any existing chunks for this file.
|
|
68
|
+
*/
|
|
69
|
+
addDocument(file: string, content: string): Promise<void>;
|
|
70
|
+
/**
|
|
71
|
+
* Search the vector store for content similar to the query.
|
|
72
|
+
* Returns top-k results sorted by cosine similarity (descending).
|
|
73
|
+
*
|
|
74
|
+
* For performance with large DBs (>10K chunks), embeddings are loaded
|
|
75
|
+
* as Float32Array and compared using optimized JS computation.
|
|
76
|
+
*/
|
|
77
|
+
search(query: string, limit?: number): Promise<VectorSearchResult[]>;
|
|
78
|
+
/**
|
|
79
|
+
* Remove all chunks for a given file.
|
|
80
|
+
*/
|
|
81
|
+
removeDocument(file: string): Promise<void>;
|
|
82
|
+
/**
|
|
83
|
+
* Full reindex from a file map (filename → content).
|
|
84
|
+
* Clears all existing data and re-indexes everything.
|
|
85
|
+
*/
|
|
86
|
+
reindex(files: Map<string, string>): Promise<void>;
|
|
87
|
+
/**
|
|
88
|
+
* Get statistics about the vector store.
|
|
89
|
+
*/
|
|
90
|
+
getStats(): {
|
|
91
|
+
documentCount: number;
|
|
92
|
+
chunkCount: number;
|
|
93
|
+
lastUpdate: string;
|
|
94
|
+
};
|
|
95
|
+
/**
|
|
96
|
+
* Close the database connection and persist to disk.
|
|
97
|
+
*/
|
|
98
|
+
close(): void;
|
|
99
|
+
}
|
|
100
|
+
//# sourceMappingURL=vector-store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vector-store.d.ts","sourceRoot":"","sources":["../src/vector-store.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAYrD;;;;;;GAMG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,EAAE,CAAyB;IACnC,OAAO,CAAC,QAAQ,CAAa;IAC7B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,WAAW,CAA8B;IACjD,OAAO,CAAC,YAAY,CAA8B;gBAEtC,MAAM,EAAE,MAAM;IAI1B;;;;OAIG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;YAQb,KAAK;IAsCnB;;;OAGG;YACW,kBAAkB;IAsBhC;;OAEG;YACW,cAAc;IAM5B;;OAEG;IACH,OAAO,CAAC,OAAO;IAOf;;OAEG;YACW,KAAK;IAOnB;;;;;;;OAOG;IACH,OAAO,CAAC,SAAS;IAiDjB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAI1B;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAO5B;;;;OAIG;IACH,OAAO,CAAC,gBAAgB;IAiBxB;;;;OAIG;IACG,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAwB/D;;;;;;OAMG;IACG,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAW,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAwC9E;;OAEG;IACG,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOjD;;;OAGG;IACG,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAcxD;;OAEG;IACH,QAAQ,IAAI;QAAE,aAAa,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE;IA2B7E;;OAEG;IACH,KAAK,IAAI,IAAI;CAUd"}
|