pigpig-agent 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.skills/code-review/SKILL.md +42 -0
- package/dist/agent/loop-detection.js +135 -0
- package/dist/agent/loop.js +123 -0
- package/dist/agent/retry.js +39 -0
- package/dist/agents/registry.js +59 -0
- package/dist/agents/spawn.js +124 -0
- package/dist/agents/types.js +5 -0
- package/dist/commands/agent.js +33 -0
- package/dist/commands/context.js +27 -0
- package/dist/commands/cron.js +38 -0
- package/dist/commands/debug.js +44 -0
- package/dist/commands/dream.js +36 -0
- package/dist/commands/index.js +10 -0
- package/dist/commands/memory.js +40 -0
- package/dist/commands/plugin.js +73 -0
- package/dist/commands/rag.js +25 -0
- package/dist/commands/security.js +44 -0
- package/dist/commands/skill.js +84 -0
- package/dist/config/init.js +69 -0
- package/dist/config/loader.js +52 -0
- package/dist/config/schema.js +55 -0
- package/dist/context/compressor.js +165 -0
- package/dist/context/defense.js +201 -0
- package/dist/context/prompt-builder.js +56 -0
- package/dist/context/prompt-pipes.js +14 -0
- package/dist/context/tool-result-output.js +25 -0
- package/dist/context/view.js +185 -0
- package/dist/cron/parser.js +27 -0
- package/dist/cron/service.js +211 -0
- package/dist/cron/store.js +53 -0
- package/dist/cron/types.js +1 -0
- package/dist/index.js +9 -0
- package/dist/main.js +270 -0
- package/dist/memory/store.js +175 -0
- package/dist/memory/validator.js +62 -0
- package/dist/mock-model.js +534 -0
- package/dist/plugins/manager.js +97 -0
- package/dist/plugins/supabase-plugin.js +111 -0
- package/dist/plugins/types.js +1 -0
- package/dist/rag/chunker.js +54 -0
- package/dist/rag/embedder.js +53 -0
- package/dist/rag/search.js +123 -0
- package/dist/rag/sqlite-store.js +195 -0
- package/dist/rag/store.js +29 -0
- package/dist/security/bash-classifier.js +39 -0
- package/dist/security/hook.js +53 -0
- package/dist/security/roles.js +16 -0
- package/dist/session/store.js +60 -0
- package/dist/skills/loader.js +100 -0
- package/dist/tools/cron-tools.js +94 -0
- package/dist/tools/file-tools.js +115 -0
- package/dist/tools/index.js +17 -0
- package/dist/tools/mcp-client.js +100 -0
- package/dist/tools/memory-tools.js +81 -0
- package/dist/tools/rag-tools.js +54 -0
- package/dist/tools/registry.js +270 -0
- package/dist/tools/search-tools.js +116 -0
- package/dist/tools/shell-tools.js +39 -0
- package/dist/tools/spawn-tools.js +35 -0
- package/dist/tools/tool-search.js +17 -0
- package/dist/tools/web-search.js +150 -0
- package/dist/usage/tracker.js +83 -0
- package/package.json +55 -0
- package/readme.md +131 -0
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
export const supabasePlugin = {
|
|
2
|
+
name: 'supabase',
|
|
3
|
+
version: '1.0.0',
|
|
4
|
+
description: '提供 Supabase 数据库操作能力(query / insert / list_tables)',
|
|
5
|
+
config: {
|
|
6
|
+
supabaseUrl: '${SUPABASE_URL}',
|
|
7
|
+
supabaseKey: '${SUPABASE_KEY}',
|
|
8
|
+
},
|
|
9
|
+
activate(api) {
|
|
10
|
+
const config = api.getConfig();
|
|
11
|
+
const url = config.supabaseUrl;
|
|
12
|
+
const key = config.supabaseKey;
|
|
13
|
+
if (!url || !key) {
|
|
14
|
+
api.log('未配置 SUPABASE_URL / SUPABASE_KEY,使用 Mock 模式');
|
|
15
|
+
}
|
|
16
|
+
api.registerTools([
|
|
17
|
+
{
|
|
18
|
+
name: 'list_tables',
|
|
19
|
+
description: '列出数据库中所有表',
|
|
20
|
+
parameters: { type: 'object', properties: {}, required: [] },
|
|
21
|
+
isConcurrencySafe: true,
|
|
22
|
+
isReadOnly: true,
|
|
23
|
+
execute: async () => {
|
|
24
|
+
if (!url) {
|
|
25
|
+
return JSON.stringify({
|
|
26
|
+
tables: ['users', 'posts', 'comments', 'sessions'],
|
|
27
|
+
note: 'Mock 模式 — 配置 SUPABASE_URL 和 SUPABASE_KEY 连接真实数据库',
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
return `连接 ${url} 查询表列表...(真实实现会调用 Supabase API)`;
|
|
31
|
+
},
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
name: 'query',
|
|
35
|
+
description: '查询指定表的数据,支持 select / where / limit',
|
|
36
|
+
parameters: {
|
|
37
|
+
type: 'object',
|
|
38
|
+
properties: {
|
|
39
|
+
table: { type: 'string', description: '表名' },
|
|
40
|
+
select: { type: 'string', description: '查询字段,默认 *' },
|
|
41
|
+
where: { type: 'string', description: '过滤条件,如 status=active' },
|
|
42
|
+
limit: { type: 'number', description: '返回条数限制,默认 10' },
|
|
43
|
+
},
|
|
44
|
+
required: ['table'],
|
|
45
|
+
},
|
|
46
|
+
isConcurrencySafe: true,
|
|
47
|
+
isReadOnly: true,
|
|
48
|
+
execute: async (input) => {
|
|
49
|
+
const { table, select = '*', where, limit = 10 } = input;
|
|
50
|
+
if (!url) {
|
|
51
|
+
const mockData = {
|
|
52
|
+
users: [
|
|
53
|
+
{ id: 1, name: '张三', email: 'zhang@example.com', role: 'admin' },
|
|
54
|
+
{ id: 2, name: '李四', email: 'li@example.com', role: 'user' },
|
|
55
|
+
{ id: 3, name: '王五', email: 'wang@example.com', role: 'user' },
|
|
56
|
+
],
|
|
57
|
+
posts: [
|
|
58
|
+
{ id: 1, title: 'Agent 开发入门', author_id: 1, status: 'published' },
|
|
59
|
+
{ id: 2, title: 'Plugin 架构设计', author_id: 1, status: 'draft' },
|
|
60
|
+
],
|
|
61
|
+
comments: [
|
|
62
|
+
{ id: 1, post_id: 1, user_id: 2, content: '写得不错!' },
|
|
63
|
+
],
|
|
64
|
+
sessions: [
|
|
65
|
+
{ id: 'sess-001', user_id: 1, created_at: '2026-05-01T10:00:00Z' },
|
|
66
|
+
],
|
|
67
|
+
};
|
|
68
|
+
const rows = mockData[table] || [];
|
|
69
|
+
let filtered = rows;
|
|
70
|
+
if (where) {
|
|
71
|
+
const [field, value] = where.split('=');
|
|
72
|
+
filtered = rows.filter(r => String(r[field]) === value);
|
|
73
|
+
}
|
|
74
|
+
return JSON.stringify({ table, rows: filtered.slice(0, limit), total: filtered.length });
|
|
75
|
+
}
|
|
76
|
+
return `SELECT ${select} FROM ${table}${where ? ` WHERE ${where}` : ''} LIMIT ${limit}`;
|
|
77
|
+
},
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
name: 'insert',
|
|
81
|
+
description: '向指定表插入一条记录',
|
|
82
|
+
parameters: {
|
|
83
|
+
type: 'object',
|
|
84
|
+
properties: {
|
|
85
|
+
table: { type: 'string', description: '表名' },
|
|
86
|
+
data: { type: 'object', description: '要插入的数据' },
|
|
87
|
+
},
|
|
88
|
+
required: ['table', 'data'],
|
|
89
|
+
},
|
|
90
|
+
isConcurrencySafe: false,
|
|
91
|
+
isReadOnly: false,
|
|
92
|
+
execute: async (input) => {
|
|
93
|
+
const { table, data } = input;
|
|
94
|
+
if (!url) {
|
|
95
|
+
return JSON.stringify({
|
|
96
|
+
success: true,
|
|
97
|
+
table,
|
|
98
|
+
inserted: { id: Math.floor(Math.random() * 1000), ...data },
|
|
99
|
+
note: 'Mock 模式',
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
return `INSERT INTO ${table} — ${JSON.stringify(data)}`;
|
|
103
|
+
},
|
|
104
|
+
},
|
|
105
|
+
]);
|
|
106
|
+
api.log(`已注册 3 个工具(list_tables / query / insert)`);
|
|
107
|
+
},
|
|
108
|
+
destroy() {
|
|
109
|
+
console.log(' [plugin:supabase] 连接已释放');
|
|
110
|
+
},
|
|
111
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
const TARGET_TOKEN = 256; // 目标token数
|
|
2
|
+
const CHARS_PER_TOKEN = 4; // 每个token的字符数
|
|
3
|
+
const TARGET_CHARS = TARGET_TOKEN * CHARS_PER_TOKEN; // 目标字符数
|
|
4
|
+
export function chunkDocument(source, text) {
|
|
5
|
+
const paragraphs = text.split(/\n{2,}/); // ['段落1', '段落2', ...]
|
|
6
|
+
const chunks = [];
|
|
7
|
+
let current = '';
|
|
8
|
+
let idx = 0;
|
|
9
|
+
for (const para of paragraphs) {
|
|
10
|
+
const trimmed = para.trim();
|
|
11
|
+
if (trimmed.length === 0)
|
|
12
|
+
continue;
|
|
13
|
+
// 缓冲区 + 新段落 > 目标字符数,先将缓冲区中的内容存起来
|
|
14
|
+
if (current.length + trimmed.length + 2 > TARGET_CHARS && current.length > 0) { // 这一段到了最大字符数
|
|
15
|
+
chunks.push(makeChunk(source, current.trim(), idx++));
|
|
16
|
+
current = '';
|
|
17
|
+
}
|
|
18
|
+
// 单个段落 > 目标字符数,按照句子分割
|
|
19
|
+
if (trimmed.length > TARGET_CHARS) { // 超长
|
|
20
|
+
if (current.length > 0) {
|
|
21
|
+
chunks.push(makeChunk(source, current.trim(), idx++));
|
|
22
|
+
current = '';
|
|
23
|
+
}
|
|
24
|
+
const sentences = trimmed.split(/(?<=[。!?.!?])\s*/);
|
|
25
|
+
let sentBuf = ''; // 句子缓冲区
|
|
26
|
+
for (const sent of sentences) {
|
|
27
|
+
if (sentBuf.length + sent.length + 1 > TARGET_CHARS && sentBuf.length > 0) {
|
|
28
|
+
chunks.push(makeChunk(source, sentBuf.trim(), idx++));
|
|
29
|
+
sentBuf = '';
|
|
30
|
+
}
|
|
31
|
+
sentBuf += (sentBuf ? ' ' : '') + sent;
|
|
32
|
+
}
|
|
33
|
+
if (sentBuf.trim()) {
|
|
34
|
+
current += sentBuf.trim();
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
else {
|
|
38
|
+
current += (current ? '\n\n' : '') + trimmed;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
if (current.trim()) {
|
|
42
|
+
chunks.push(makeChunk(source, current.trim(), idx++));
|
|
43
|
+
}
|
|
44
|
+
return chunks;
|
|
45
|
+
}
|
|
46
|
+
function makeChunk(source, text, index) {
|
|
47
|
+
return {
|
|
48
|
+
id: `${source}#${index}`,
|
|
49
|
+
text,
|
|
50
|
+
source,
|
|
51
|
+
index,
|
|
52
|
+
tokenEstimate: Math.ceil(text.length / CHARS_PER_TOKEN),
|
|
53
|
+
};
|
|
54
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
const DIMS = 128; // 128维度
|
|
2
|
+
export function createDashScopeEmbedder(apiKey) {
|
|
3
|
+
return async (texts) => {
|
|
4
|
+
const resp = await fetch('https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings', {
|
|
5
|
+
method: 'POST',
|
|
6
|
+
headers: {
|
|
7
|
+
Authorization: `Bearer ${apiKey}`,
|
|
8
|
+
'Content-Type': 'application/json',
|
|
9
|
+
},
|
|
10
|
+
body: JSON.stringify({
|
|
11
|
+
model: 'text-embedding-v3',
|
|
12
|
+
input: texts,
|
|
13
|
+
dimensions: DIMS,
|
|
14
|
+
}),
|
|
15
|
+
});
|
|
16
|
+
const data = await resp.json();
|
|
17
|
+
return data.data.map((d) => d.embedding);
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
// 相同的文本没必要重复做向量处理
|
|
21
|
+
const embedCache = new Map();
|
|
22
|
+
export async function embed(fn, texts) {
|
|
23
|
+
const results = new Array(texts.length);
|
|
24
|
+
const uncached = [];
|
|
25
|
+
for (let i = 0; i < texts.length; i++) {
|
|
26
|
+
const cached = embedCache.get(texts[i]);
|
|
27
|
+
if (cached) {
|
|
28
|
+
results[i] = cached;
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
uncached.push({ idx: i, text: texts[i] });
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
if (uncached.length > 0) {
|
|
35
|
+
const vectors = await fn(uncached.map(u => u.text)); // 批量处理未缓存的文本
|
|
36
|
+
for (let i = 0; i < uncached.length; i++) {
|
|
37
|
+
results[uncached[i].idx] = vectors[i];
|
|
38
|
+
embedCache.set(uncached[i].text, vectors[i]);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
return results;
|
|
42
|
+
}
|
|
43
|
+
// 计算向量之间的余弦相似度 --- 两个向量”有多像“
|
|
44
|
+
export function cosineSimilarity(a, b) {
|
|
45
|
+
let dot = 0, normA = 0, normB = 0;
|
|
46
|
+
for (let i = 0; i < a.length; i++) {
|
|
47
|
+
dot += a[i] * b[i];
|
|
48
|
+
normA += a[i] * a[i];
|
|
49
|
+
normB += b[i] * b[i];
|
|
50
|
+
}
|
|
51
|
+
return dot / (Math.sqrt(normA) * Math.sqrt(normB) || 1);
|
|
52
|
+
}
|
|
53
|
+
export { DIMS };
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { cosineSimilarity, embed } from "./embedder.js";
|
|
2
|
+
const VECTOR_WEIGHT = 0.7;
|
|
3
|
+
const KEYWORD_WEIGHT = 0.3;
|
|
4
|
+
const CANIDATE_MULTIPLIER = 4; // top-k 候选数量
|
|
5
|
+
const MMR_LAMBDA = 0.7; // MMR 算法的参数,70% 相关性 30% 多样性
|
|
6
|
+
export async function hybridSearch(// 混合检索
|
|
7
|
+
store, embedFn, query, topK = 5) {
|
|
8
|
+
const all = store.getAll(); // 获取所有的向量数据
|
|
9
|
+
const candidateCount = Math.min(topK * CANIDATE_MULTIPLIER, all.length);
|
|
10
|
+
// 路径一:向量检索
|
|
11
|
+
const [queryVec] = await embed(embedFn, [query]); // 先将查询语句转成向量
|
|
12
|
+
const vectorResults = all
|
|
13
|
+
.map(chunk => ({ chunk, score: cosineSimilarity(queryVec, chunk.embedding) }))
|
|
14
|
+
.sort((a, b) => b.score - a.score)
|
|
15
|
+
.slice(0, candidateCount);
|
|
16
|
+
// 路径二:关键词检索 (BM25 算法)
|
|
17
|
+
const queryTerms = tokenize(query); // 对查询语句进行分词
|
|
18
|
+
const docCount = all.length; // 文档数量
|
|
19
|
+
const keywordResults = all
|
|
20
|
+
.map(chunk => ({ chunk, score: bm25Score(queryTerms, chunk.text, docCount, all) }))
|
|
21
|
+
.sort((a, b) => b.score - a.score)
|
|
22
|
+
.slice(0, candidateCount);
|
|
23
|
+
// 归一化:不是两种分数的和
|
|
24
|
+
const vecNorm = normalizeMinMax(vectorResults.map(r => r.score));
|
|
25
|
+
const kwNorm = normalizeViaSigmoid(keywordResults.map(r => r.score)); // 因为BM25 得到的分数可能是负数也可能很大
|
|
26
|
+
// 合并结果
|
|
27
|
+
const candidates = new Map();
|
|
28
|
+
for (let i = 0; i < vectorResults.length; i++) {
|
|
29
|
+
const id = vectorResults[i].chunk.id;
|
|
30
|
+
candidates.set(id, {
|
|
31
|
+
chunk: vectorResults[i].chunk,
|
|
32
|
+
score: vecNorm[i] * VECTOR_WEIGHT,
|
|
33
|
+
vectorScore: vecNorm[i],
|
|
34
|
+
keywordScore: 0,
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
for (let i = 0; i < keywordResults.length; i++) {
|
|
38
|
+
const id = keywordResults[i].chunk.id;
|
|
39
|
+
const existing = candidates.get(id);
|
|
40
|
+
if (existing) {
|
|
41
|
+
existing.keywordScore = kwNorm[i];
|
|
42
|
+
existing.score += kwNorm[i] * KEYWORD_WEIGHT;
|
|
43
|
+
}
|
|
44
|
+
else {
|
|
45
|
+
candidates.set(id, {
|
|
46
|
+
chunk: keywordResults[i].chunk,
|
|
47
|
+
score: kwNorm[i] * KEYWORD_WEIGHT,
|
|
48
|
+
vectorScore: 0,
|
|
49
|
+
keywordScore: kwNorm[i],
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
// 排序
|
|
54
|
+
const sorted = [...candidates.values()].sort((a, b) => b.score - a.score);
|
|
55
|
+
// 处理数据的相关性和多样性 (防止搜到的数据有重复)
|
|
56
|
+
return mmrSelect(sorted, topK);
|
|
57
|
+
}
|
|
58
|
+
// BM25 算文本的匹配度
|
|
59
|
+
function tokenize(text) {
|
|
60
|
+
return text.toLowerCase()
|
|
61
|
+
.replace(/[^\w一-鿿]+/g, ' ')
|
|
62
|
+
.split(/\s+/)
|
|
63
|
+
.filter(t => t.length > 1);
|
|
64
|
+
}
|
|
65
|
+
function bm25Score(queryTerms, docText, N, allDocs) {
|
|
66
|
+
const k1 = 1.2;
|
|
67
|
+
const b = 0.75;
|
|
68
|
+
const docTokens = tokenize(docText);
|
|
69
|
+
const avgDl = allDocs.reduce((s, d) => s + tokenize(d.text).length, 0) / (N || 1);
|
|
70
|
+
const dl = docTokens.length;
|
|
71
|
+
let score = 0;
|
|
72
|
+
for (const term of queryTerms) {
|
|
73
|
+
const tf = docTokens.filter(t => t === term).length;
|
|
74
|
+
const df = allDocs.filter(d => tokenize(d.text).includes(term)).length;
|
|
75
|
+
const idf = Math.log((N - df + 0.5) / (df + 0.5) + 1);
|
|
76
|
+
const tfNorm = (tf * (k1 + 1)) / (tf + k1 * (1 - b + b * (dl / avgDl)));
|
|
77
|
+
score += idf * tfNorm;
|
|
78
|
+
}
|
|
79
|
+
return score;
|
|
80
|
+
}
|
|
81
|
+
// 归一化操作
|
|
82
|
+
function normalizeMinMax(scores) {
|
|
83
|
+
if (scores.length === 0)
|
|
84
|
+
return [];
|
|
85
|
+
const min = Math.min(...scores);
|
|
86
|
+
const max = Math.max(...scores);
|
|
87
|
+
const range = max - min || 1;
|
|
88
|
+
return scores.map(s => (s - min) / range);
|
|
89
|
+
}
|
|
90
|
+
// 计算 sigmoid 函数的值,将分数映射到 (0, 1) 范围内,同时保持分数的相对顺序
|
|
91
|
+
function normalizeViaSigmoid(scores) {
|
|
92
|
+
return scores.map(s => 1 / (1 + Math.exp(-s)));
|
|
93
|
+
}
|
|
94
|
+
// MMR 去重
|
|
95
|
+
function mmrSelect(results, topK) {
|
|
96
|
+
if (results.length <= topK)
|
|
97
|
+
return results;
|
|
98
|
+
const selected = [results[0]];
|
|
99
|
+
const remaining = results.slice(1);
|
|
100
|
+
while (selected.length < topK && remaining.length > 0) {
|
|
101
|
+
let bestIdx = 0;
|
|
102
|
+
let bestMmr = -Infinity;
|
|
103
|
+
for (let i = 0; i < remaining.length; i++) {
|
|
104
|
+
const relevance = remaining[i].score;
|
|
105
|
+
const maxSim = Math.max(...selected.map(s => jaccardSimilarity(s.chunk.text, remaining[i].chunk.text)));
|
|
106
|
+
const mmr = MMR_LAMBDA * relevance - (1 - MMR_LAMBDA) * maxSim;
|
|
107
|
+
if (mmr > bestMmr) {
|
|
108
|
+
bestMmr = mmr;
|
|
109
|
+
bestIdx = i;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
selected.push(remaining[bestIdx]);
|
|
113
|
+
remaining.splice(bestIdx, 1);
|
|
114
|
+
}
|
|
115
|
+
return selected;
|
|
116
|
+
}
|
|
117
|
+
function jaccardSimilarity(a, b) {
|
|
118
|
+
const setA = new Set(tokenize(a));
|
|
119
|
+
const setB = new Set(tokenize(b));
|
|
120
|
+
const intersection = [...setA].filter(t => setB.has(t)).length;
|
|
121
|
+
const union = new Set([...setA, ...setB]).size;
|
|
122
|
+
return union === 0 ? 0 : intersection / union;
|
|
123
|
+
}
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
import Database from 'better-sqlite3';
|
|
2
|
+
import * as sqliteVec from 'sqlite-vec';
|
|
3
|
+
import { embed } from './embedder.js';
|
|
4
|
+
const MMR_LAMBDA = 0.7;
|
|
5
|
+
export class SqliteVectorStore {
|
|
6
|
+
db;
|
|
7
|
+
constructor(dbPath = 'knowledge.db') {
|
|
8
|
+
this.db = new Database(dbPath);
|
|
9
|
+
sqliteVec.load(this.db); // 加载向量搜索扩展
|
|
10
|
+
this.createTables();
|
|
11
|
+
}
|
|
12
|
+
createTables() {
|
|
13
|
+
this.db.exec(`
|
|
14
|
+
CREATE TABLE IF NOT EXISTS chunks (
|
|
15
|
+
id TEXT PRIMARY KEY,
|
|
16
|
+
text TEXT NOT NULL,
|
|
17
|
+
source TEXT NOT NULL,
|
|
18
|
+
chunk_index INTEGER NOT NULL,
|
|
19
|
+
embedding TEXT NOT NULL,
|
|
20
|
+
model TEXT NOT NULL DEFAULT 'text-embedding-v3',
|
|
21
|
+
updated_at INTEGER NOT NULL
|
|
22
|
+
);
|
|
23
|
+
|
|
24
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_vec USING vec0(
|
|
25
|
+
id TEXT PRIMARY KEY,
|
|
26
|
+
embedding FLOAT[128]
|
|
27
|
+
);
|
|
28
|
+
|
|
29
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
|
|
30
|
+
text, id UNINDEXED, source UNINDEXED
|
|
31
|
+
);
|
|
32
|
+
`);
|
|
33
|
+
}
|
|
34
|
+
add(chunk, embedding) {
|
|
35
|
+
const now = Date.now();
|
|
36
|
+
// 三表联动写入
|
|
37
|
+
this.db.prepare(`INSERT OR REPLACE INTO chunks
|
|
38
|
+
(id, text, source, chunk_index, embedding, updated_at)
|
|
39
|
+
VALUES (?, ?, ?, ?, ?, ?)`)
|
|
40
|
+
.run(chunk.id, chunk.text, chunk.source, chunk.index, JSON.stringify(embedding), now);
|
|
41
|
+
this.db.prepare(`INSERT OR REPLACE INTO chunks_vec (id, embedding)
|
|
42
|
+
VALUES (?, ?)`)
|
|
43
|
+
.run(chunk.id, Buffer.from(new Float32Array(embedding).buffer));
|
|
44
|
+
this.db.prepare(`INSERT OR REPLACE INTO chunks_fts (id, text, source)
|
|
45
|
+
VALUES (?, ?, ?)`)
|
|
46
|
+
.run(chunk.id, chunk.text, chunk.source);
|
|
47
|
+
}
|
|
48
|
+
addBatch(items) {
|
|
49
|
+
const tx = this.db.transaction(() => {
|
|
50
|
+
for (const { chunk, embedding } of items)
|
|
51
|
+
this.add(chunk, embedding);
|
|
52
|
+
});
|
|
53
|
+
tx(); // 事务批量写入,比逐条快很多
|
|
54
|
+
}
|
|
55
|
+
vectorSearch(queryEmbedding, topK) {
|
|
56
|
+
const buf = Buffer.from(new Float32Array(queryEmbedding).buffer);
|
|
57
|
+
const rows = this.db.prepare(`
|
|
58
|
+
SELECT v.id, v.distance, c.text, c.source, c.chunk_index, c.embedding
|
|
59
|
+
FROM chunks_vec v
|
|
60
|
+
JOIN chunks c ON c.id = v.id
|
|
61
|
+
WHERE v.embedding MATCH ?
|
|
62
|
+
ORDER BY v.distance
|
|
63
|
+
LIMIT ?
|
|
64
|
+
`).all(buf, topK);
|
|
65
|
+
return rows.map(r => ({
|
|
66
|
+
chunk: {
|
|
67
|
+
id: r.id, text: r.text, source: r.source,
|
|
68
|
+
index: r.chunk_index,
|
|
69
|
+
tokenEstimate: Math.ceil(r.text.length / 4),
|
|
70
|
+
embedding: JSON.parse(r.embedding),
|
|
71
|
+
addedAt: 0,
|
|
72
|
+
},
|
|
73
|
+
score: 1 - r.distance, // cosine distance → similarity
|
|
74
|
+
}));
|
|
75
|
+
}
|
|
76
|
+
keywordSearch(query, topK) {
|
|
77
|
+
const rows = this.db.prepare(`
|
|
78
|
+
SELECT f.id, bm25(chunks_fts) AS rank, c.text, c.source, c.chunk_index, c.embedding
|
|
79
|
+
FROM chunks_fts f
|
|
80
|
+
JOIN chunks c ON c.id = f.id
|
|
81
|
+
WHERE chunks_fts MATCH ?
|
|
82
|
+
ORDER BY rank
|
|
83
|
+
LIMIT ?
|
|
84
|
+
`).all(query, topK);
|
|
85
|
+
return rows.map(r => ({
|
|
86
|
+
chunk: {
|
|
87
|
+
id: r.id, text: r.text, source: r.source,
|
|
88
|
+
index: r.chunk_index,
|
|
89
|
+
tokenEstimate: Math.ceil(r.text.length / 4),
|
|
90
|
+
embedding: JSON.parse(r.embedding),
|
|
91
|
+
addedAt: 0,
|
|
92
|
+
},
|
|
93
|
+
score: r.rank < 0 ? -r.rank / (1 - r.rank) : 1 / (1 + r.rank),
|
|
94
|
+
}));
|
|
95
|
+
}
|
|
96
|
+
size() {
|
|
97
|
+
return this.db.prepare('SELECT COUNT(*) as n FROM chunks').get().n;
|
|
98
|
+
}
|
|
99
|
+
clear() {
|
|
100
|
+
this.db.exec('DELETE FROM chunks; DELETE FROM chunks_vec; DELETE FROM chunks_fts;');
|
|
101
|
+
}
|
|
102
|
+
sources() {
|
|
103
|
+
return this.db.prepare('SELECT DISTINCT source FROM chunks').all().map(r => r.source);
|
|
104
|
+
}
|
|
105
|
+
// 混合搜索:直接在 SQLite 层完成向量 + 关键词双路检索
|
|
106
|
+
async hybridSearch(embedFn, query, topK = 5) {
|
|
107
|
+
const candidateCount = Math.min(topK * 4, this.size());
|
|
108
|
+
if (candidateCount === 0)
|
|
109
|
+
return [];
|
|
110
|
+
const [queryVec] = await embed(embedFn, [query]);
|
|
111
|
+
// 路径 1: sqlite-vec 向量搜索
|
|
112
|
+
const vectorResults = this.vectorSearch(queryVec, candidateCount);
|
|
113
|
+
// 路径 2: FTS5 关键词搜索
|
|
114
|
+
const keywordResults = this.keywordSearch(query, candidateCount);
|
|
115
|
+
// 归一化 + 加权合并
|
|
116
|
+
const vecScores = normalizeMinMax(vectorResults.map(r => r.score));
|
|
117
|
+
const kwScores = normalizeMinMax(keywordResults.map(r => r.score));
|
|
118
|
+
const candidates = new Map();
|
|
119
|
+
for (let i = 0; i < vectorResults.length; i++) {
|
|
120
|
+
const id = vectorResults[i].chunk.id;
|
|
121
|
+
candidates.set(id, {
|
|
122
|
+
chunk: vectorResults[i].chunk,
|
|
123
|
+
score: vecScores[i] * 0.7,
|
|
124
|
+
vectorScore: vecScores[i],
|
|
125
|
+
keywordScore: 0,
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
for (let i = 0; i < keywordResults.length; i++) {
|
|
129
|
+
const id = keywordResults[i].chunk.id;
|
|
130
|
+
const existing = candidates.get(id);
|
|
131
|
+
if (existing) {
|
|
132
|
+
existing.keywordScore = kwScores[i];
|
|
133
|
+
existing.score += kwScores[i] * 0.3;
|
|
134
|
+
}
|
|
135
|
+
else {
|
|
136
|
+
candidates.set(id, {
|
|
137
|
+
chunk: keywordResults[i].chunk,
|
|
138
|
+
score: kwScores[i] * 0.3,
|
|
139
|
+
vectorScore: 0,
|
|
140
|
+
keywordScore: kwScores[i],
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
const sorted = [...candidates.values()]
|
|
145
|
+
.sort((a, b) => b.score - a.score);
|
|
146
|
+
// MMR deduplication
|
|
147
|
+
return mmrSelect(sorted, topK);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
// BM25 算文本的匹配度
|
|
151
|
+
function tokenize(text) {
|
|
152
|
+
return text.toLowerCase()
|
|
153
|
+
.replace(/[^\w一-鿿]+/g, ' ')
|
|
154
|
+
.split(/\s+/)
|
|
155
|
+
.filter(t => t.length > 1);
|
|
156
|
+
}
|
|
157
|
+
// 归一化操作
|
|
158
|
+
function normalizeMinMax(scores) {
|
|
159
|
+
if (scores.length === 0)
|
|
160
|
+
return [];
|
|
161
|
+
const min = Math.min(...scores);
|
|
162
|
+
const max = Math.max(...scores);
|
|
163
|
+
const range = max - min || 1;
|
|
164
|
+
return scores.map(s => (s - min) / range);
|
|
165
|
+
}
|
|
166
|
+
// MMR 去重
|
|
167
|
+
function mmrSelect(results, topK) {
|
|
168
|
+
if (results.length <= topK)
|
|
169
|
+
return results;
|
|
170
|
+
const selected = [results[0]];
|
|
171
|
+
const remaining = results.slice(1);
|
|
172
|
+
while (selected.length < topK && remaining.length > 0) {
|
|
173
|
+
let bestIdx = 0;
|
|
174
|
+
let bestMmr = -Infinity;
|
|
175
|
+
for (let i = 0; i < remaining.length; i++) {
|
|
176
|
+
const relevance = remaining[i].score;
|
|
177
|
+
const maxSim = Math.max(...selected.map(s => jaccardSimilarity(s.chunk.text, remaining[i].chunk.text)));
|
|
178
|
+
const mmr = MMR_LAMBDA * relevance - (1 - MMR_LAMBDA) * maxSim;
|
|
179
|
+
if (mmr > bestMmr) {
|
|
180
|
+
bestMmr = mmr;
|
|
181
|
+
bestIdx = i;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
selected.push(remaining[bestIdx]);
|
|
185
|
+
remaining.splice(bestIdx, 1);
|
|
186
|
+
}
|
|
187
|
+
return selected;
|
|
188
|
+
}
|
|
189
|
+
function jaccardSimilarity(a, b) {
|
|
190
|
+
const setA = new Set(tokenize(a));
|
|
191
|
+
const setB = new Set(tokenize(b));
|
|
192
|
+
const intersection = [...setA].filter(t => setB.has(t)).length;
|
|
193
|
+
const union = new Set([...setA, ...setB]).size;
|
|
194
|
+
return union === 0 ? 0 : intersection / union;
|
|
195
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export class VectorStore {
|
|
2
|
+
chunks = []; // 自己打造的向量数据库
|
|
3
|
+
add(chunk, embedding) {
|
|
4
|
+
const existing = this.chunks.findIndex(c => c.id === chunk.id);
|
|
5
|
+
if (existing >= 0) {
|
|
6
|
+
this.chunks[existing] = { ...chunk, embedding, addedAt: Date.now() };
|
|
7
|
+
}
|
|
8
|
+
else {
|
|
9
|
+
this.chunks.push({ ...chunk, embedding, addedAt: Date.now() });
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
addBatch(items) {
|
|
13
|
+
for (const { chunk, embedding } of items) {
|
|
14
|
+
this.add(chunk, embedding);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
getAll() {
|
|
18
|
+
return this.chunks;
|
|
19
|
+
}
|
|
20
|
+
size() {
|
|
21
|
+
return this.chunks.length;
|
|
22
|
+
}
|
|
23
|
+
clear() {
|
|
24
|
+
this.chunks = [];
|
|
25
|
+
}
|
|
26
|
+
sources() {
|
|
27
|
+
return [...new Set(this.chunks.map(c => c.source))];
|
|
28
|
+
}
|
|
29
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
const DANGEROUS_PATTERNS = [
|
|
2
|
+
{ pattern: /\brm\s+(-[a-zA-Z]*f[a-zA-Z]*\s+|.*-rf\b|.*--force)/, reason: '强制删除文件' },
|
|
3
|
+
{ pattern: /\brm\s+-[a-zA-Z]*r/, reason: '递归删除' },
|
|
4
|
+
{ pattern: /\bsudo\b/, reason: '提权操作' },
|
|
5
|
+
{ pattern: /\bmkfs\b/, reason: '格式化磁盘' },
|
|
6
|
+
{ pattern: /\bdd\s+.*of=\/dev\//, reason: '直接写设备' },
|
|
7
|
+
{ pattern: /:\(\)\s*\{.*\|.*&\s*\}/, reason: 'Fork bomb' },
|
|
8
|
+
{ pattern: />\s*\/dev\/sd[a-z]/, reason: '覆写磁盘设备' },
|
|
9
|
+
{ pattern: /\bchmod\s+777\b/, reason: '开放所有权限' },
|
|
10
|
+
{ pattern: /\bcurl\b.*\|\s*(ba)?sh/, reason: '远程脚本执行' },
|
|
11
|
+
{ pattern: /\bwget\b.*\|\s*(ba)?sh/, reason: '远程脚本执行' },
|
|
12
|
+
{ pattern: /\beval\b/, reason: 'eval 动态执行' },
|
|
13
|
+
{ pattern: />\s*\/etc\//, reason: '覆写系统配置' },
|
|
14
|
+
];
|
|
15
|
+
const MODERATE_PATTERNS = [
|
|
16
|
+
{ pattern: /\brm\b/, reason: '删除文件' },
|
|
17
|
+
{ pattern: /\bmv\b/, reason: '移动/重命名文件' },
|
|
18
|
+
{ pattern: /\bchmod\b/, reason: '修改权限' },
|
|
19
|
+
{ pattern: /\bchown\b/, reason: '修改所有者' },
|
|
20
|
+
{ pattern: /\bkill\b/, reason: '终止进程' },
|
|
21
|
+
{ pattern: /\bpkill\b/, reason: '批量终止进程' },
|
|
22
|
+
{ pattern: /\bgit\s+push\b/, reason: 'Git 推送' },
|
|
23
|
+
{ pattern: /\bgit\s+reset\s+--hard\b/, reason: 'Git 硬重置' },
|
|
24
|
+
{ pattern: /\bnpm\s+publish\b/, reason: '发布 npm 包' },
|
|
25
|
+
{ pattern: /\bdocker\s+rm\b/, reason: '删除容器' },
|
|
26
|
+
];
|
|
27
|
+
export function classifyBashCommand(command) {
|
|
28
|
+
for (const { pattern, reason } of DANGEROUS_PATTERNS) {
|
|
29
|
+
if (pattern.test(command)) {
|
|
30
|
+
return { level: 'dangerous', reason };
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
for (const { pattern, reason } of MODERATE_PATTERNS) {
|
|
34
|
+
if (pattern.test(command)) {
|
|
35
|
+
return { level: 'moderate', reason };
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return { level: 'safe' };
|
|
39
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
export class HookPipeline {
|
|
2
|
+
preHooks = [];
|
|
3
|
+
postHooks = [];
|
|
4
|
+
registerPre(name, fn) {
|
|
5
|
+
this.preHooks.push({ name, fn });
|
|
6
|
+
}
|
|
7
|
+
registerPost(name, fn) {
|
|
8
|
+
this.postHooks.push({ name, fn });
|
|
9
|
+
}
|
|
10
|
+
async runPre(toolName, input) {
|
|
11
|
+
let currentInput = input;
|
|
12
|
+
for (const hook of this.preHooks) {
|
|
13
|
+
try {
|
|
14
|
+
const result = await hook.fn(toolName, currentInput);
|
|
15
|
+
if (result.action === 'block') {
|
|
16
|
+
console.log(` [hook: ${hook.name}] 拦截 ${toolName}: ${result.reason}`);
|
|
17
|
+
return result;
|
|
18
|
+
}
|
|
19
|
+
if (result.action === 'modify' && result.modifiedInput !== undefined) {
|
|
20
|
+
currentInput = result.modifiedInput;
|
|
21
|
+
console.log(` [hook: ${hook.name}] 修改了 ${toolName} 的输入`);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
catch (err) {
|
|
25
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
26
|
+
console.error(` [hook: ${hook.name}] 执行Pre异常: ${msg}`);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
return { action: 'allow' };
|
|
30
|
+
}
|
|
31
|
+
async runPost(toolName, input, output) {
|
|
32
|
+
let currentOutput = output;
|
|
33
|
+
for (const hook of this.postHooks) {
|
|
34
|
+
try {
|
|
35
|
+
const result = await hook.fn(toolName, input, currentOutput);
|
|
36
|
+
if (result.action === 'modify' && result.modifiedOutput !== undefined) {
|
|
37
|
+
currentOutput = result.modifiedOutput;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
catch (err) {
|
|
41
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
42
|
+
console.error(` [hook: ${hook.name}] 执行Post异常: ${msg}`);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return { action: 'allow', modifiedOutput: currentOutput };
|
|
46
|
+
}
|
|
47
|
+
list() {
|
|
48
|
+
return {
|
|
49
|
+
pre: this.preHooks.map(hook => hook.name),
|
|
50
|
+
post: this.postHooks.map(hook => hook.name)
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
const TOOL_ACCESS = {
|
|
2
|
+
owner: { allow: '*', deny: [] }, // 完全访问
|
|
3
|
+
collaborator: { allow: '*', deny: ['bash'] }, // 自动审批
|
|
4
|
+
guest: { allow: ['read_file', 'list_directory', 'glob', 'grep', 'rag_search'], deny: [] }, // 手动审批
|
|
5
|
+
};
|
|
6
|
+
export function canUseTool(role, toolName) {
|
|
7
|
+
const access = TOOL_ACCESS[role];
|
|
8
|
+
if (access.deny.includes(toolName))
|
|
9
|
+
return false;
|
|
10
|
+
if (access.allow === '*')
|
|
11
|
+
return true;
|
|
12
|
+
return access.allow.includes(toolName);
|
|
13
|
+
}
|
|
14
|
+
export function filterToolsForRole(toolNames, role) {
|
|
15
|
+
return toolNames.filter(name => canUseTool(role, name));
|
|
16
|
+
}
|