promptgraph-mcp 2.8.1 → 2.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +205 -205
- package/ann.js +33 -33
- package/api.js +202 -202
- package/bundle-counts.js +111 -111
- package/chunker.js +28 -28
- package/cli.js +115 -115
- package/commands/bundle.js +150 -150
- package/commands/doctor.js +15 -15
- package/commands/import.js +7 -7
- package/commands/init.js +37 -37
- package/commands/marketplace.js +146 -146
- package/commands/reindex.js +10 -10
- package/commands/search.js +55 -55
- package/commands/setup.js +19 -19
- package/commands/status.js +110 -110
- package/commands/train.js +18 -18
- package/commands/update.js +49 -49
- package/commands/validate.js +63 -63
- package/config.js +72 -72
- package/db.js +157 -157
- package/doctor.js +48 -48
- package/embedder.js +54 -54
- package/github-import.js +750 -745
- package/indexer.js +310 -310
- package/package.json +61 -61
- package/parser.js +69 -69
- package/pg-hook.js +70 -70
- package/platform.js +120 -120
- package/search.js +216 -216
- package/src/filter/classifier.js +88 -88
- package/src/filter/hard-filter.js +62 -62
- package/src/filter/train.js +66 -66
- package/src/reranker/reranker.js +92 -92
- package/src/store/flat-store.js +61 -61
- package/src/store/hnsw-store.js +187 -187
- package/src/store/index.js +19 -19
- package/src/store/vector-store.js +9 -9
- package/src/utils/rate-limiter.js +33 -33
- package/tui.js +418 -418
- package/validate-repo-action.js +139 -139
- package/watcher.js +84 -84
package/db.js
CHANGED
|
@@ -1,157 +1,157 @@
|
|
|
1
|
-
import Database from 'better-sqlite3';
|
|
2
|
-
import path from 'path';
|
|
3
|
-
import fs from 'fs';
|
|
4
|
-
import { PROMPTGRAPH_DIR } from './config.js';
|
|
5
|
-
|
|
6
|
-
const DB_PATH = path.join(PROMPTGRAPH_DIR, 'promptgraph.db');
|
|
7
|
-
|
|
8
|
-
let _db = null;
|
|
9
|
-
|
|
10
|
-
const MIGRATIONS = [
|
|
11
|
-
{
|
|
12
|
-
version: 1, description: 'initial schema',
|
|
13
|
-
up: db => db.exec(`
|
|
14
|
-
CREATE TABLE IF NOT EXISTS skills (
|
|
15
|
-
id TEXT PRIMARY KEY,
|
|
16
|
-
name TEXT NOT NULL,
|
|
17
|
-
description TEXT,
|
|
18
|
-
path TEXT NOT NULL,
|
|
19
|
-
source TEXT NOT NULL,
|
|
20
|
-
content TEXT NOT NULL,
|
|
21
|
-
hash TEXT
|
|
22
|
-
);
|
|
23
|
-
CREATE TABLE IF NOT EXISTS chunks (
|
|
24
|
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
25
|
-
skill_id TEXT NOT NULL,
|
|
26
|
-
chunk_index INTEGER NOT NULL,
|
|
27
|
-
text TEXT NOT NULL,
|
|
28
|
-
embedding BLOB NOT NULL,
|
|
29
|
-
UNIQUE(skill_id, chunk_index)
|
|
30
|
-
);
|
|
31
|
-
CREATE TABLE IF NOT EXISTS edges (
|
|
32
|
-
from_skill TEXT NOT NULL,
|
|
33
|
-
to_skill TEXT NOT NULL,
|
|
34
|
-
PRIMARY KEY (from_skill, to_skill)
|
|
35
|
-
);
|
|
36
|
-
CREATE TABLE IF NOT EXISTS ratings (
|
|
37
|
-
skill_id TEXT PRIMARY KEY,
|
|
38
|
-
uses INTEGER DEFAULT 0,
|
|
39
|
-
success INTEGER DEFAULT 0,
|
|
40
|
-
fail INTEGER DEFAULT 0
|
|
41
|
-
);
|
|
42
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS skills_fts USING fts5(
|
|
43
|
-
id UNINDEXED, name, description, content,
|
|
44
|
-
content='skills', content_rowid='rowid'
|
|
45
|
-
);
|
|
46
|
-
`),
|
|
47
|
-
},
|
|
48
|
-
{
|
|
49
|
-
version: 2, description: 'add hash column',
|
|
50
|
-
up: db => {
|
|
51
|
-
const cols = db.pragma('table_info(skills)').map(c => c.name);
|
|
52
|
-
if (!cols.includes('hash')) db.exec('ALTER TABLE skills ADD COLUMN hash TEXT');
|
|
53
|
-
},
|
|
54
|
-
},
|
|
55
|
-
{
|
|
56
|
-
version: 3, description: 'add registry metadata columns',
|
|
57
|
-
up: db => {
|
|
58
|
-
const cols = db.pragma('table_info(skills)').map(c => c.name);
|
|
59
|
-
for (const [col, def] of [
|
|
60
|
-
['version', 'TEXT'],
|
|
61
|
-
['author', 'TEXT'],
|
|
62
|
-
['license', 'TEXT'],
|
|
63
|
-
['updated_at', 'TEXT'],
|
|
64
|
-
['downloads', 'INTEGER DEFAULT 0'],
|
|
65
|
-
['verified', 'INTEGER DEFAULT 0'],
|
|
66
|
-
['trust_level', "TEXT DEFAULT 'unknown'"],
|
|
67
|
-
['rating', 'REAL DEFAULT 0'],
|
|
68
|
-
['rating_count', 'INTEGER DEFAULT 0'],
|
|
69
|
-
['popularity', 'REAL DEFAULT 0'],
|
|
70
|
-
['last_update', 'TEXT'],
|
|
71
|
-
]) {
|
|
72
|
-
if (!cols.includes(col)) db.exec(`ALTER TABLE skills ADD COLUMN ${col} ${def}`);
|
|
73
|
-
}
|
|
74
|
-
},
|
|
75
|
-
},
|
|
76
|
-
{
|
|
77
|
-
version: 4, description: 'registry_entries table',
|
|
78
|
-
up: db => db.exec(`
|
|
79
|
-
CREATE TABLE IF NOT EXISTS registry_entries (
|
|
80
|
-
id TEXT PRIMARY KEY,
|
|
81
|
-
trust_level TEXT DEFAULT 'unknown',
|
|
82
|
-
downloads INTEGER DEFAULT 0,
|
|
83
|
-
rating REAL DEFAULT 0,
|
|
84
|
-
rating_count INTEGER DEFAULT 0,
|
|
85
|
-
popularity REAL DEFAULT 0,
|
|
86
|
-
last_update TEXT
|
|
87
|
-
);
|
|
88
|
-
`),
|
|
89
|
-
},
|
|
90
|
-
{
|
|
91
|
-
version: 5, description: 'convert TEXT embeddings to Float32 BLOB',
|
|
92
|
-
up: db => {
|
|
93
|
-
const textEmbeddings = db.prepare("SELECT COUNT(*) as n FROM chunks WHERE typeof(embedding) = 'text'").get();
|
|
94
|
-
if (textEmbeddings?.n > 0) {
|
|
95
|
-
const rows = db.prepare("SELECT rowid, embedding FROM chunks WHERE typeof(embedding) = 'text'").all();
|
|
96
|
-
const upd = db.prepare('UPDATE chunks SET embedding = ? WHERE rowid = ?');
|
|
97
|
-
db.transaction(() => {
|
|
98
|
-
for (const row of rows) {
|
|
99
|
-
const vec = JSON.parse(row.embedding);
|
|
100
|
-
upd.run(Buffer.from(new Float32Array(vec).buffer), row.rowid);
|
|
101
|
-
}
|
|
102
|
-
})();
|
|
103
|
-
console.error(`[PromptGraph] Migrated ${textEmbeddings.n} embeddings TEXT→BLOB`);
|
|
104
|
-
}
|
|
105
|
-
},
|
|
106
|
-
},
|
|
107
|
-
]
|
|
108
|
-
|
|
109
|
-
function getSchemaVersion(db) {
|
|
110
|
-
try {
|
|
111
|
-
return db.prepare('SELECT MAX(version) as v FROM _schema_version').get().v || 0
|
|
112
|
-
} catch {
|
|
113
|
-
return 0
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
function runMigrations(db) {
|
|
118
|
-
const current = getSchemaVersion(db)
|
|
119
|
-
const pending = MIGRATIONS.filter(m => m.version > current).sort((a, b) => a.version - b.version)
|
|
120
|
-
if (!pending.length) return
|
|
121
|
-
|
|
122
|
-
db.exec(`CREATE TABLE IF NOT EXISTS _schema_version (version INTEGER PRIMARY KEY, description TEXT, applied_at TEXT)`)
|
|
123
|
-
|
|
124
|
-
for (const migration of pending) {
|
|
125
|
-
db.transaction(() => {
|
|
126
|
-
migration.up(db)
|
|
127
|
-
db.prepare('INSERT INTO _schema_version (version, description, applied_at) VALUES (?, ?, ?)').run(
|
|
128
|
-
migration.version, migration.description, new Date().toISOString()
|
|
129
|
-
)
|
|
130
|
-
})()
|
|
131
|
-
console.error(`[PromptGraph] DB migrated to v${migration.version}: ${migration.description}`)
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
export function getDb() {
|
|
136
|
-
if (_db) return _db;
|
|
137
|
-
fs.mkdirSync(path.dirname(DB_PATH), { recursive: true });
|
|
138
|
-
const db = new Database(DB_PATH);
|
|
139
|
-
_db = db;
|
|
140
|
-
db.pragma('journal_mode = WAL');
|
|
141
|
-
runMigrations(db);
|
|
142
|
-
return db;
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
export function skillId(source, name) {
|
|
146
|
-
return `${source}::${name}`;
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
export function vecToBlob(vec) {
|
|
150
|
-
return Buffer.from(new Float32Array(vec).buffer);
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
export function blobToVec(blob) {
|
|
154
|
-
if (typeof blob === 'string') return JSON.parse(blob);
|
|
155
|
-
const buf = Buffer.isBuffer(blob) ? blob : Buffer.from(blob);
|
|
156
|
-
return Array.from(new Float32Array(buf.buffer, buf.byteOffset, buf.length / 4));
|
|
157
|
-
}
|
|
1
|
+
import Database from 'better-sqlite3';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import fs from 'fs';
|
|
4
|
+
import { PROMPTGRAPH_DIR } from './config.js';
|
|
5
|
+
|
|
6
|
+
const DB_PATH = path.join(PROMPTGRAPH_DIR, 'promptgraph.db');
|
|
7
|
+
|
|
8
|
+
let _db = null;
|
|
9
|
+
|
|
10
|
+
const MIGRATIONS = [
|
|
11
|
+
{
|
|
12
|
+
version: 1, description: 'initial schema',
|
|
13
|
+
up: db => db.exec(`
|
|
14
|
+
CREATE TABLE IF NOT EXISTS skills (
|
|
15
|
+
id TEXT PRIMARY KEY,
|
|
16
|
+
name TEXT NOT NULL,
|
|
17
|
+
description TEXT,
|
|
18
|
+
path TEXT NOT NULL,
|
|
19
|
+
source TEXT NOT NULL,
|
|
20
|
+
content TEXT NOT NULL,
|
|
21
|
+
hash TEXT
|
|
22
|
+
);
|
|
23
|
+
CREATE TABLE IF NOT EXISTS chunks (
|
|
24
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
25
|
+
skill_id TEXT NOT NULL,
|
|
26
|
+
chunk_index INTEGER NOT NULL,
|
|
27
|
+
text TEXT NOT NULL,
|
|
28
|
+
embedding BLOB NOT NULL,
|
|
29
|
+
UNIQUE(skill_id, chunk_index)
|
|
30
|
+
);
|
|
31
|
+
CREATE TABLE IF NOT EXISTS edges (
|
|
32
|
+
from_skill TEXT NOT NULL,
|
|
33
|
+
to_skill TEXT NOT NULL,
|
|
34
|
+
PRIMARY KEY (from_skill, to_skill)
|
|
35
|
+
);
|
|
36
|
+
CREATE TABLE IF NOT EXISTS ratings (
|
|
37
|
+
skill_id TEXT PRIMARY KEY,
|
|
38
|
+
uses INTEGER DEFAULT 0,
|
|
39
|
+
success INTEGER DEFAULT 0,
|
|
40
|
+
fail INTEGER DEFAULT 0
|
|
41
|
+
);
|
|
42
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS skills_fts USING fts5(
|
|
43
|
+
id UNINDEXED, name, description, content,
|
|
44
|
+
content='skills', content_rowid='rowid'
|
|
45
|
+
);
|
|
46
|
+
`),
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
version: 2, description: 'add hash column',
|
|
50
|
+
up: db => {
|
|
51
|
+
const cols = db.pragma('table_info(skills)').map(c => c.name);
|
|
52
|
+
if (!cols.includes('hash')) db.exec('ALTER TABLE skills ADD COLUMN hash TEXT');
|
|
53
|
+
},
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
version: 3, description: 'add registry metadata columns',
|
|
57
|
+
up: db => {
|
|
58
|
+
const cols = db.pragma('table_info(skills)').map(c => c.name);
|
|
59
|
+
for (const [col, def] of [
|
|
60
|
+
['version', 'TEXT'],
|
|
61
|
+
['author', 'TEXT'],
|
|
62
|
+
['license', 'TEXT'],
|
|
63
|
+
['updated_at', 'TEXT'],
|
|
64
|
+
['downloads', 'INTEGER DEFAULT 0'],
|
|
65
|
+
['verified', 'INTEGER DEFAULT 0'],
|
|
66
|
+
['trust_level', "TEXT DEFAULT 'unknown'"],
|
|
67
|
+
['rating', 'REAL DEFAULT 0'],
|
|
68
|
+
['rating_count', 'INTEGER DEFAULT 0'],
|
|
69
|
+
['popularity', 'REAL DEFAULT 0'],
|
|
70
|
+
['last_update', 'TEXT'],
|
|
71
|
+
]) {
|
|
72
|
+
if (!cols.includes(col)) db.exec(`ALTER TABLE skills ADD COLUMN ${col} ${def}`);
|
|
73
|
+
}
|
|
74
|
+
},
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
version: 4, description: 'registry_entries table',
|
|
78
|
+
up: db => db.exec(`
|
|
79
|
+
CREATE TABLE IF NOT EXISTS registry_entries (
|
|
80
|
+
id TEXT PRIMARY KEY,
|
|
81
|
+
trust_level TEXT DEFAULT 'unknown',
|
|
82
|
+
downloads INTEGER DEFAULT 0,
|
|
83
|
+
rating REAL DEFAULT 0,
|
|
84
|
+
rating_count INTEGER DEFAULT 0,
|
|
85
|
+
popularity REAL DEFAULT 0,
|
|
86
|
+
last_update TEXT
|
|
87
|
+
);
|
|
88
|
+
`),
|
|
89
|
+
},
|
|
90
|
+
{
|
|
91
|
+
version: 5, description: 'convert TEXT embeddings to Float32 BLOB',
|
|
92
|
+
up: db => {
|
|
93
|
+
const textEmbeddings = db.prepare("SELECT COUNT(*) as n FROM chunks WHERE typeof(embedding) = 'text'").get();
|
|
94
|
+
if (textEmbeddings?.n > 0) {
|
|
95
|
+
const rows = db.prepare("SELECT rowid, embedding FROM chunks WHERE typeof(embedding) = 'text'").all();
|
|
96
|
+
const upd = db.prepare('UPDATE chunks SET embedding = ? WHERE rowid = ?');
|
|
97
|
+
db.transaction(() => {
|
|
98
|
+
for (const row of rows) {
|
|
99
|
+
const vec = JSON.parse(row.embedding);
|
|
100
|
+
upd.run(Buffer.from(new Float32Array(vec).buffer), row.rowid);
|
|
101
|
+
}
|
|
102
|
+
})();
|
|
103
|
+
console.error(`[PromptGraph] Migrated ${textEmbeddings.n} embeddings TEXT→BLOB`);
|
|
104
|
+
}
|
|
105
|
+
},
|
|
106
|
+
},
|
|
107
|
+
]
|
|
108
|
+
|
|
109
|
+
function getSchemaVersion(db) {
|
|
110
|
+
try {
|
|
111
|
+
return db.prepare('SELECT MAX(version) as v FROM _schema_version').get().v || 0
|
|
112
|
+
} catch {
|
|
113
|
+
return 0
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function runMigrations(db) {
|
|
118
|
+
const current = getSchemaVersion(db)
|
|
119
|
+
const pending = MIGRATIONS.filter(m => m.version > current).sort((a, b) => a.version - b.version)
|
|
120
|
+
if (!pending.length) return
|
|
121
|
+
|
|
122
|
+
db.exec(`CREATE TABLE IF NOT EXISTS _schema_version (version INTEGER PRIMARY KEY, description TEXT, applied_at TEXT)`)
|
|
123
|
+
|
|
124
|
+
for (const migration of pending) {
|
|
125
|
+
db.transaction(() => {
|
|
126
|
+
migration.up(db)
|
|
127
|
+
db.prepare('INSERT INTO _schema_version (version, description, applied_at) VALUES (?, ?, ?)').run(
|
|
128
|
+
migration.version, migration.description, new Date().toISOString()
|
|
129
|
+
)
|
|
130
|
+
})()
|
|
131
|
+
console.error(`[PromptGraph] DB migrated to v${migration.version}: ${migration.description}`)
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export function getDb() {
|
|
136
|
+
if (_db) return _db;
|
|
137
|
+
fs.mkdirSync(path.dirname(DB_PATH), { recursive: true });
|
|
138
|
+
const db = new Database(DB_PATH);
|
|
139
|
+
_db = db;
|
|
140
|
+
db.pragma('journal_mode = WAL');
|
|
141
|
+
runMigrations(db);
|
|
142
|
+
return db;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export function skillId(source, name) {
|
|
146
|
+
return `${source}::${name}`;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export function vecToBlob(vec) {
|
|
150
|
+
return Buffer.from(new Float32Array(vec).buffer);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
export function blobToVec(blob) {
|
|
154
|
+
if (typeof blob === 'string') return JSON.parse(blob);
|
|
155
|
+
const buf = Buffer.isBuffer(blob) ? blob : Buffer.from(blob);
|
|
156
|
+
return Array.from(new Float32Array(buf.buffer, buf.byteOffset, buf.length / 4));
|
|
157
|
+
}
|
package/doctor.js
CHANGED
|
@@ -1,48 +1,48 @@
|
|
|
1
|
-
import { getDb } from './db.js';
|
|
2
|
-
|
|
3
|
-
export function runDoctor() {
|
|
4
|
-
const db = getDb();
|
|
5
|
-
const report = {};
|
|
6
|
-
|
|
7
|
-
// orphaned chunks (skill_id not in skills)
|
|
8
|
-
const orphanChunks = db.prepare(`
|
|
9
|
-
DELETE FROM chunks WHERE skill_id NOT IN (SELECT id FROM skills)
|
|
10
|
-
`).run();
|
|
11
|
-
report.orphanChunks = orphanChunks.changes;
|
|
12
|
-
|
|
13
|
-
// orphaned ratings
|
|
14
|
-
const orphanRatings = db.prepare(`
|
|
15
|
-
DELETE FROM ratings WHERE skill_id NOT IN (SELECT id FROM skills)
|
|
16
|
-
`).run();
|
|
17
|
-
report.orphanRatings = orphanRatings.changes;
|
|
18
|
-
|
|
19
|
-
// orphaned edges where from_skill no longer exists
|
|
20
|
-
const orphanFromEdges = db.prepare(`
|
|
21
|
-
DELETE FROM edges WHERE from_skill NOT IN (SELECT id FROM skills)
|
|
22
|
-
`).run();
|
|
23
|
-
report.orphanFromEdges = orphanFromEdges.changes;
|
|
24
|
-
|
|
25
|
-
// dangling edges where to_skill is a bare name that never resolved to a real skill
|
|
26
|
-
// (keep edges that point to real ids OR bare names that match a skill name)
|
|
27
|
-
const danglingEdges = db.prepare(`
|
|
28
|
-
DELETE FROM edges
|
|
29
|
-
WHERE to_skill NOT IN (SELECT id FROM skills)
|
|
30
|
-
AND to_skill NOT IN (SELECT name FROM skills)
|
|
31
|
-
`).run();
|
|
32
|
-
report.danglingEdges = danglingEdges.changes;
|
|
33
|
-
|
|
34
|
-
// duplicate skills by path (should not happen, but check)
|
|
35
|
-
const dupPaths = db.prepare(`
|
|
36
|
-
SELECT path, COUNT(*) as c FROM skills GROUP BY path HAVING c > 1
|
|
37
|
-
`).all();
|
|
38
|
-
report.duplicatePaths = dupPaths.length;
|
|
39
|
-
|
|
40
|
-
db.pragma('wal_checkpoint(TRUNCATE)');
|
|
41
|
-
db.exec('VACUUM');
|
|
42
|
-
|
|
43
|
-
report.totalSkills = db.prepare('SELECT COUNT(*) as c FROM skills').get().c;
|
|
44
|
-
report.totalChunks = db.prepare('SELECT COUNT(*) as c FROM chunks').get().c;
|
|
45
|
-
report.totalEdges = db.prepare('SELECT COUNT(*) as c FROM edges').get().c;
|
|
46
|
-
|
|
47
|
-
return report;
|
|
48
|
-
}
|
|
1
|
+
import { getDb } from './db.js';
|
|
2
|
+
|
|
3
|
+
export function runDoctor() {
|
|
4
|
+
const db = getDb();
|
|
5
|
+
const report = {};
|
|
6
|
+
|
|
7
|
+
// orphaned chunks (skill_id not in skills)
|
|
8
|
+
const orphanChunks = db.prepare(`
|
|
9
|
+
DELETE FROM chunks WHERE skill_id NOT IN (SELECT id FROM skills)
|
|
10
|
+
`).run();
|
|
11
|
+
report.orphanChunks = orphanChunks.changes;
|
|
12
|
+
|
|
13
|
+
// orphaned ratings
|
|
14
|
+
const orphanRatings = db.prepare(`
|
|
15
|
+
DELETE FROM ratings WHERE skill_id NOT IN (SELECT id FROM skills)
|
|
16
|
+
`).run();
|
|
17
|
+
report.orphanRatings = orphanRatings.changes;
|
|
18
|
+
|
|
19
|
+
// orphaned edges where from_skill no longer exists
|
|
20
|
+
const orphanFromEdges = db.prepare(`
|
|
21
|
+
DELETE FROM edges WHERE from_skill NOT IN (SELECT id FROM skills)
|
|
22
|
+
`).run();
|
|
23
|
+
report.orphanFromEdges = orphanFromEdges.changes;
|
|
24
|
+
|
|
25
|
+
// dangling edges where to_skill is a bare name that never resolved to a real skill
|
|
26
|
+
// (keep edges that point to real ids OR bare names that match a skill name)
|
|
27
|
+
const danglingEdges = db.prepare(`
|
|
28
|
+
DELETE FROM edges
|
|
29
|
+
WHERE to_skill NOT IN (SELECT id FROM skills)
|
|
30
|
+
AND to_skill NOT IN (SELECT name FROM skills)
|
|
31
|
+
`).run();
|
|
32
|
+
report.danglingEdges = danglingEdges.changes;
|
|
33
|
+
|
|
34
|
+
// duplicate skills by path (should not happen, but check)
|
|
35
|
+
const dupPaths = db.prepare(`
|
|
36
|
+
SELECT path, COUNT(*) as c FROM skills GROUP BY path HAVING c > 1
|
|
37
|
+
`).all();
|
|
38
|
+
report.duplicatePaths = dupPaths.length;
|
|
39
|
+
|
|
40
|
+
db.pragma('wal_checkpoint(TRUNCATE)');
|
|
41
|
+
db.exec('VACUUM');
|
|
42
|
+
|
|
43
|
+
report.totalSkills = db.prepare('SELECT COUNT(*) as c FROM skills').get().c;
|
|
44
|
+
report.totalChunks = db.prepare('SELECT COUNT(*) as c FROM chunks').get().c;
|
|
45
|
+
report.totalEdges = db.prepare('SELECT COUNT(*) as c FROM edges').get().c;
|
|
46
|
+
|
|
47
|
+
return report;
|
|
48
|
+
}
|
package/embedder.js
CHANGED
|
@@ -1,54 +1,54 @@
|
|
|
1
|
-
import { EmbeddingModel, FlagEmbedding } from 'fastembed';
|
|
2
|
-
import path from 'path';
|
|
3
|
-
import os from 'os';
|
|
4
|
-
|
|
5
|
-
const CACHE_DIR = path.join(os.homedir(), '.claude', '.promptgraph', 'model-cache');
|
|
6
|
-
const BATCH_SIZE = 256;
|
|
7
|
-
const MAX_EMBEDDING_CALLS = 10000;
|
|
8
|
-
let embedCallCount = 0;
|
|
9
|
-
|
|
10
|
-
export function getEmbedCallCount() { return embedCallCount; }
|
|
11
|
-
export function resetEmbedCallCount() { embedCallCount = 0; }
|
|
12
|
-
|
|
13
|
-
let model = null;
|
|
14
|
-
|
|
15
|
-
async function getModel() {
|
|
16
|
-
if (!model) {
|
|
17
|
-
model = await FlagEmbedding.init({
|
|
18
|
-
model: EmbeddingModel.BGESmallENV15,
|
|
19
|
-
cacheDir: CACHE_DIR,
|
|
20
|
-
});
|
|
21
|
-
}
|
|
22
|
-
return model;
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
export async function embed(text) {
|
|
26
|
-
embedCallCount++;
|
|
27
|
-
const m = await getModel();
|
|
28
|
-
const results = [];
|
|
29
|
-
for await (const batch of m.embed([text])) {
|
|
30
|
-
results.push(...batch);
|
|
31
|
-
}
|
|
32
|
-
return Array.from(results[0]);
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
export async function embedBatch(texts) {
|
|
36
|
-
if (embedCallCount + texts.length > MAX_EMBEDDING_CALLS) {
|
|
37
|
-
throw new Error(`Embedding queue limit exceeded (max ${MAX_EMBEDDING_CALLS} chunks per session). Use --fast or reindex incrementally.`);
|
|
38
|
-
}
|
|
39
|
-
embedCallCount += texts.length;
|
|
40
|
-
const m = await getModel();
|
|
41
|
-
const all = [];
|
|
42
|
-
for await (const batch of m.embed(texts)) {
|
|
43
|
-
all.push(...batch);
|
|
44
|
-
}
|
|
45
|
-
return all.map(v => Array.from(v));
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
export function cosineSimilarity(a, b) {
|
|
49
|
-
let dot = 0;
|
|
50
|
-
for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
|
|
51
|
-
return dot;
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
export { BATCH_SIZE };
|
|
1
|
+
import { EmbeddingModel, FlagEmbedding } from 'fastembed';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import os from 'os';
|
|
4
|
+
|
|
5
|
+
const CACHE_DIR = path.join(os.homedir(), '.claude', '.promptgraph', 'model-cache');
|
|
6
|
+
const BATCH_SIZE = 256;
|
|
7
|
+
const MAX_EMBEDDING_CALLS = 10000;
|
|
8
|
+
let embedCallCount = 0;
|
|
9
|
+
|
|
10
|
+
export function getEmbedCallCount() { return embedCallCount; }
|
|
11
|
+
export function resetEmbedCallCount() { embedCallCount = 0; }
|
|
12
|
+
|
|
13
|
+
let model = null;
|
|
14
|
+
|
|
15
|
+
async function getModel() {
|
|
16
|
+
if (!model) {
|
|
17
|
+
model = await FlagEmbedding.init({
|
|
18
|
+
model: EmbeddingModel.BGESmallENV15,
|
|
19
|
+
cacheDir: CACHE_DIR,
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
return model;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export async function embed(text) {
|
|
26
|
+
embedCallCount++;
|
|
27
|
+
const m = await getModel();
|
|
28
|
+
const results = [];
|
|
29
|
+
for await (const batch of m.embed([text])) {
|
|
30
|
+
results.push(...batch);
|
|
31
|
+
}
|
|
32
|
+
return Array.from(results[0]);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export async function embedBatch(texts) {
|
|
36
|
+
if (embedCallCount + texts.length > MAX_EMBEDDING_CALLS) {
|
|
37
|
+
throw new Error(`Embedding queue limit exceeded (max ${MAX_EMBEDDING_CALLS} chunks per session). Use --fast or reindex incrementally.`);
|
|
38
|
+
}
|
|
39
|
+
embedCallCount += texts.length;
|
|
40
|
+
const m = await getModel();
|
|
41
|
+
const all = [];
|
|
42
|
+
for await (const batch of m.embed(texts)) {
|
|
43
|
+
all.push(...batch);
|
|
44
|
+
}
|
|
45
|
+
return all.map(v => Array.from(v));
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function cosineSimilarity(a, b) {
|
|
49
|
+
let dot = 0;
|
|
50
|
+
for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
|
|
51
|
+
return dot;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export { BATCH_SIZE };
|