skillar 0.2.1 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +280 -0
- package/README.md +37 -5
- package/dist/analysis/finding.js +8 -190
- package/dist/analysis/kernel.js +896 -0
- package/dist/analysis/tree.js +7 -154
- package/dist/analysis/usage.js +9 -394
- package/dist/api/flows.js +42 -212
- package/dist/api/report-kernel.js +1063 -0
- package/dist/api/routes.js +12 -2
- package/dist/catalog/fields.js +138 -146
- package/dist/catalog/watcher.js +7 -2
- package/dist/cli/init.js +45 -0
- package/dist/cli/main.js +76 -3
- package/dist/cli/private-skills.js +122 -0
- package/dist/cli/serve.js +15 -1
- package/dist/cli/sync.js +124 -0
- package/dist/dashboard/NOTICE +280 -0
- package/dist/dashboard/assets/app.js +737 -275
- package/dist/dashboard/assets/astryx-reset.css +407 -0
- package/dist/dashboard/assets/astryx.css +1077 -0
- package/dist/dashboard/assets/charts.css +54 -0
- package/dist/dashboard/assets/chunk-TPGDA2J4.js +1 -0
- package/dist/dashboard/assets/chunk-YLWHVDMP.js +36 -0
- package/dist/dashboard/assets/controls.css +207 -0
- package/dist/dashboard/assets/overlays.css +132 -0
- package/dist/dashboard/assets/p1.css +135 -0
- package/dist/dashboard/assets/shell.css +71 -0
- package/dist/dashboard/assets/skillar-theme.css +692 -0
- package/dist/dashboard/assets/styles.css +224 -121
- package/dist/dashboard/assets/tokens.css +278 -0
- package/dist/dashboard/assets/widgets.css +83 -0
- package/dist/dashboard/index.html +11 -2
- package/dist/dashboard/viewers/viewers.css +61 -0
- package/dist/ingest/attribution.js +5 -1
- package/dist/ingest/pipeline.js +2 -0
- package/dist/ingest/request-recorder.js +39 -0
- package/dist/mcp/dispatch.js +2 -2
- package/dist/mcp/http.js +43 -3
- package/dist/mcp/skills-extension.js +42 -15
- package/dist/mcp/stdio-proxy.js +24 -3
- package/dist/mcp/tools.js +38 -24
- package/dist/packs/claude-code.js +22 -11
- package/dist/packs/cursor.js +9 -4
- package/dist/packs/gemini.js +9 -4
- package/dist/packs/json-config.js +41 -0
- package/dist/packs/omo.js +15 -6
- package/dist/request-status.js +81 -0
- package/dist/search/fts.js +93 -58
- package/dist/search/fusion.js +49 -47
- package/dist/search/guidance.js +18 -6
- package/dist/search/index.js +291 -70
- package/dist/search/lexical.js +71 -63
- package/dist/server.js +12 -0
- package/dist/skills/apply.js +130 -0
- package/dist/skills/capture.js +54 -0
- package/dist/skills/cloud-plan.js +57 -0
- package/dist/skills/cloud.js +81 -0
- package/dist/skills/fs.js +133 -0
- package/dist/skills/package.js +734 -0
- package/dist/skills/plan.js +83 -0
- package/dist/skills/receipt.js +44 -0
- package/dist/skills/rollback.js +119 -0
- package/dist/skills/transfer.js +51 -0
- package/dist/store/db.js +7 -1
- package/dist/store/execution.js +2 -34
- package/dist/store/export.js +25 -4
- package/dist/store/migrations/007_sync.sql +49 -0
- package/dist/store/migrations/008_connector_commands.sql +12 -0
- package/dist/store/queries.js +33 -7
- package/dist/store/writes.js +38 -5
- package/dist/sync/cloud-blobs.js +175 -0
- package/dist/sync/cloud-commands.js +101 -0
- package/dist/sync/cloud-connection.js +105 -0
- package/dist/sync/cloud-delivery.js +114 -0
- package/dist/sync/cloud-documents.js +103 -0
- package/dist/sync/cloud-export.js +67 -0
- package/dist/sync/cloud-http.js +150 -0
- package/dist/sync/cloud-jobs.js +94 -0
- package/dist/sync/cloud-pending.js +143 -0
- package/dist/sync/cloud-prepare.js +101 -0
- package/dist/sync/cloud-restore.js +136 -0
- package/dist/sync/cloud-rollback.js +24 -0
- package/dist/sync/cloud-runner.js +86 -0
- package/dist/sync/cloud-runtime.js +293 -0
- package/dist/sync/cloud-schema.js +32 -0
- package/dist/sync/cloud-stage.js +112 -0
- package/dist/sync/cloud-wire.js +693 -0
- package/dist/sync/cloud-worker.js +57 -0
- package/dist/sync/cloud.js +157 -0
- package/dist/sync/command-apply.js +140 -0
- package/dist/sync/command-control.js +25 -0
- package/dist/sync/command-wire.js +437 -0
- package/dist/sync/consent.js +130 -0
- package/dist/sync/journal.js +43 -0
- package/dist/sync/outbox.js +134 -0
- package/dist/sync/restore-rows.js +125 -0
- package/dist/sync/restore.js +174 -0
- package/dist/sync/snapshot.js +78 -0
- package/dist/sync/spool-request.js +103 -0
- package/dist/sync/tables.js +27 -0
- package/dist/sync/triggers.js +34 -0
- package/dist/workflow/compare.js +3 -43
- package/docs/cloud-sync.md +94 -0
- package/docs/private-skills.md +75 -0
- package/docs/sync-p4.md +102 -0
- package/docs/sync-restore.md +167 -0
- package/package.json +18 -5
- package/src/store/migrations/007_sync.sql +49 -0
- package/src/store/migrations/008_connector_commands.sql +12 -0
- package/dist/dashboard/assets/chunk-5DFWNJC2.js +0 -1
- package/dist/dashboard/assets/chunk-S6B2HJQA.js +0 -17
package/dist/search/index.js
CHANGED
|
@@ -1,74 +1,295 @@
|
|
|
1
|
+
// src/search/index.ts
|
|
1
2
|
import { randomUUID } from "node:crypto";
|
|
2
3
|
import { performance } from "node:perf_hooks";
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
4
|
+
|
|
5
|
+
// ../shared/dist/search/fusion.js
|
|
6
|
+
var RRF_K = 60;
|
|
7
|
+
var DEFAULT_THRESHOLD = 0;
|
|
8
|
+
var MAX_RRF = 2 / (RRF_K + 1);
|
|
9
|
+
var round4 = (n) => Math.round(n * 1e4) / 1e4;
|
|
10
|
+
function termCoverage(queryTerms, matched) {
|
|
11
|
+
const set = new Set(matched.map((m) => m.normalize("NFKC").toLowerCase()));
|
|
12
|
+
return { hit: queryTerms.filter((t) => set.has(t)).length, total: queryTerms.length };
|
|
13
|
+
}
|
|
14
|
+
function fuse(lexical, fts, queryTerms = []) {
|
|
15
|
+
const byUri = /* @__PURE__ */ new Map();
|
|
16
|
+
const get = (uri) => {
|
|
17
|
+
let f = byUri.get(uri);
|
|
18
|
+
if (!f)
|
|
19
|
+
byUri.set(uri, f = { uri, score: 0, rrf: 0, anchor: 0, lexScore: 0, matched: [], coverage: 1 });
|
|
20
|
+
return f;
|
|
21
|
+
};
|
|
22
|
+
lexical.forEach((hit, i) => {
|
|
23
|
+
const f = get(hit.uri);
|
|
24
|
+
f.rrf += 1 / (RRF_K + i + 1);
|
|
25
|
+
f.anchor = hit.anchor;
|
|
26
|
+
f.lexScore = hit.score;
|
|
27
|
+
f.matched.push(...hit.matched);
|
|
28
|
+
});
|
|
29
|
+
for (const hit of fts) {
|
|
30
|
+
const f = get(hit.uri);
|
|
31
|
+
f.rrf += 1 / (RRF_K + hit.rank);
|
|
32
|
+
f.matched.push(...hit.matched);
|
|
33
|
+
}
|
|
34
|
+
const fused = [...byUri.values()];
|
|
35
|
+
for (const f of fused) {
|
|
36
|
+
f.matched = [...new Set(f.matched)];
|
|
37
|
+
const { hit, total } = termCoverage(queryTerms, f.matched);
|
|
38
|
+
f.coverage = total ? hit / total : 1;
|
|
39
|
+
f.score = f.anchor ? 1 : round4(Math.min(1, f.rrf / MAX_RRF) * f.coverage);
|
|
40
|
+
}
|
|
41
|
+
return fused.sort((a, b) => b.anchor - a.anchor || b.rrf - a.rrf || b.lexScore - a.lexScore || (a.uri < b.uri ? -1 : a.uri > b.uri ? 1 : 0));
|
|
42
|
+
}
|
|
43
|
+
function calibrate(negativeBestScores, margin = 0.01) {
|
|
44
|
+
const hits = negativeBestScores.filter((s) => s > 0);
|
|
45
|
+
return hits.length ? round4(Math.min(1, Math.max(...hits) + margin)) : 0;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// ../shared/dist/search/guidance.js
|
|
49
|
+
var SEARCH_TOOL_GUIDANCE = "Rewrite the user's task as a short English intent sentence using skill vocabulary (Korean is accepted). Do not pass skill bodies. Use mode=discovery.";
|
|
50
|
+
var ABSTAIN_HINTS = {
|
|
51
|
+
empty_query: "Describe the task as an intent sentence",
|
|
52
|
+
no_overlap: "No skill shares vocabulary with this query; rewrite it with skill vocabulary or continue without a skill",
|
|
53
|
+
below_threshold: "Matches are too weak; rephrase as a short intent sentence or continue without a skill",
|
|
54
|
+
explicit_only: "Only explicit-invocation skills matched; ask the user or search with mode=explicit"
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
// ../shared/dist/search/lexical.js
|
|
58
|
+
var normalize = (value) => value.normalize("NFKC").toLowerCase().replace(/\s+/g, " ").trim();
|
|
59
|
+
var STOPWORDS = new Set("a an the and or to of for in on with from by is are be it this that do please use using when how me my can could would should not only then after before need want help \uD574\uC918 \uD574\uC8FC\uC138\uC694 \uC54C\uB824\uC918 \uC791\uC131\uD574\uC918 \uB9CC\uB4E4\uC5B4\uC918 \uD655\uC778\uD574\uC918 \uAC80\uD1A0\uD574\uC918 \uD544\uC694 \uC5C6\uC5B4 \uC788\uC2B5\uB2C8\uB2E4 \uD569\uB2C8\uB2E4 \uC791\uC5C5 \uC0AC\uC6A9 \uD1B5\uD574 \uB300\uD55C \uC704\uD55C \uAD00\uB828 \uD604\uC7AC \uB2E4\uC74C \uBA3C\uC800 \uC2E4\uC81C \uD558\uB098 \uAC83 \uC218 \uBC0F \uB610\uB294 \uD558\uACE0 \uADF8\uB9AC\uACE0 \uBC29\uBC95 \uC740 \uB294 \uC774 \uAC00 \uC744 \uB97C \uC758 \uB3C4 \uB85C \uC640 \uACFC \uC5D0".split(" "));
|
|
60
|
+
function stem(word) {
|
|
61
|
+
if (/^[가-힣]+$/.test(word) && word.length > 2) {
|
|
62
|
+
const stripped = word.replace(/(?:해주세요|해줘|하기|하게|하는|하면서|하려는|으로|에서|에게|에는|까지|부터|처럼|보다|은|는|을|를|이|가|과|와|의|도|로)$/, "");
|
|
63
|
+
return stripped.length >= 2 ? stripped : word;
|
|
64
|
+
}
|
|
65
|
+
return /^[a-z]{5,}s$/.test(word) && !word.endsWith("ss") ? word.slice(0, -1) : word;
|
|
66
|
+
}
|
|
67
|
+
function words(text) {
|
|
68
|
+
return (normalize(text).match(/[\p{L}\p{N}]+/gu) ?? []).flatMap((word) => word.split(/(?<=[가-힣])(?=[a-z0-9])|(?<=[a-z0-9])(?=[가-힣])/));
|
|
69
|
+
}
|
|
70
|
+
var isStopword = (word) => STOPWORDS.has(word);
|
|
71
|
+
function terms(text) {
|
|
72
|
+
return [...new Set(words(text).filter((word) => !STOPWORDS.has(word)).map(stem).filter((word) => !STOPWORDS.has(word)))];
|
|
73
|
+
}
|
|
74
|
+
function keywordsOf(entry) {
|
|
75
|
+
return [...entry.examples, ...entry.aliasesKo, ...entry.whenToUse ? terms(entry.whenToUse) : []].map(normalize).filter(Boolean);
|
|
76
|
+
}
|
|
77
|
+
function buildLexical(entries) {
|
|
78
|
+
const frequency = /* @__PURE__ */ new Map();
|
|
79
|
+
const rows = entries.map((entry) => {
|
|
80
|
+
const name = normalize(entry.name), keywords = keywordsOf(entry);
|
|
81
|
+
const weights = /* @__PURE__ */ new Map();
|
|
82
|
+
for (const [field, weight] of [[entry.description, 1], [keywords.join(" "), 4], [name, 8]]) {
|
|
83
|
+
for (const term of terms(field))
|
|
84
|
+
weights.set(term, weight);
|
|
85
|
+
}
|
|
86
|
+
for (const term of weights.keys())
|
|
87
|
+
frequency.set(term, (frequency.get(term) ?? 0) + 1);
|
|
88
|
+
return { uri: entry.uri, name, keywords, nameTerms: new Set(terms(name)), weights };
|
|
89
|
+
});
|
|
90
|
+
const size = rows.length;
|
|
91
|
+
return {
|
|
92
|
+
search(query) {
|
|
93
|
+
const text = normalize(query);
|
|
94
|
+
if (!text)
|
|
95
|
+
return [];
|
|
96
|
+
const queryTerms = terms(text);
|
|
97
|
+
const hits = [];
|
|
98
|
+
for (const { uri, name, keywords, nameTerms, weights } of rows) {
|
|
99
|
+
const anchor = name === text ? 2 : keywords.includes(text) ? 1 : 0;
|
|
100
|
+
let score = anchor === 2 ? 1e3 : anchor === 1 ? 300 : 0;
|
|
101
|
+
let anchored = anchor > 0;
|
|
102
|
+
const matched = [];
|
|
103
|
+
for (const word of queryTerms) {
|
|
104
|
+
const weight = weights.get(word);
|
|
105
|
+
if (!weight)
|
|
106
|
+
continue;
|
|
107
|
+
matched.push(word);
|
|
108
|
+
anchored ||= nameTerms.has(word) || keywords.includes(word);
|
|
109
|
+
const rarity = Math.log(1 + (size + 0.5) / ((frequency.get(word) ?? 0) + 0.5));
|
|
110
|
+
score += weight * rarity;
|
|
30
111
|
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
112
|
+
if (!score || queryTerms.length > 1 && matched.length === 1 && !anchored)
|
|
113
|
+
continue;
|
|
114
|
+
score *= 1 + matched.length / Math.max(queryTerms.length, 1);
|
|
115
|
+
hits.push({ uri, score, matched, anchor });
|
|
116
|
+
}
|
|
117
|
+
return hits.sort((a, b) => b.score - a.score || (a.uri < b.uri ? -1 : a.uri > b.uri ? 1 : 0));
|
|
118
|
+
}
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// ../shared/dist/search/core.js
|
|
123
|
+
function buildSearchCore(list, fts) {
|
|
124
|
+
const entries = new Map(list.map((e) => [e.uri, e]));
|
|
125
|
+
const lexical = buildLexical([...entries.values()]);
|
|
126
|
+
return (query, opts, threshold = DEFAULT_THRESHOLD) => {
|
|
127
|
+
const abstain = (reason) => ({ candidates: [], abstained: { reason, hint: ABSTAIN_HINTS[reason] } });
|
|
128
|
+
const text = normalize(query ?? "");
|
|
129
|
+
if (!text)
|
|
130
|
+
return abstain("empty_query");
|
|
131
|
+
const fused = fuse(lexical.search(text), fts.search(text), terms(text)).filter((f) => entries.has(f.uri));
|
|
132
|
+
if (!fused.length)
|
|
133
|
+
return abstain("no_overlap");
|
|
134
|
+
const allowed = fused.flatMap((f) => {
|
|
135
|
+
const entry = entries.get(f.uri);
|
|
136
|
+
return entry && (opts.mode !== "automatic" || entry.invocation === "automatic") ? [{ f, entry }] : [];
|
|
137
|
+
});
|
|
138
|
+
const first = allowed[0];
|
|
139
|
+
if (!first)
|
|
140
|
+
return abstain("explicit_only");
|
|
141
|
+
if (first.f.score < threshold)
|
|
142
|
+
return abstain("below_threshold");
|
|
143
|
+
const limit = Math.max(1, Math.floor(opts.limit));
|
|
144
|
+
return { candidates: allowed.slice(0, limit).map(({ f, entry: e }, i) => ({
|
|
145
|
+
uri: e.uri,
|
|
146
|
+
name: e.name,
|
|
147
|
+
description: e.description,
|
|
148
|
+
score: f.score,
|
|
149
|
+
rank: i + 1,
|
|
150
|
+
invocation: e.invocation,
|
|
151
|
+
digest: e.digest,
|
|
152
|
+
matched: f.matched
|
|
153
|
+
})) };
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// ../shared/dist/search/fts.js
|
|
158
|
+
var FTS_MIN_CHARS = 3;
|
|
159
|
+
var length = (s) => Array.from(s).length;
|
|
160
|
+
function ftsTokens(query) {
|
|
161
|
+
const text = normalize(query);
|
|
162
|
+
if (length(text) < FTS_MIN_CHARS)
|
|
163
|
+
return [];
|
|
164
|
+
const tokens = /* @__PURE__ */ new Set();
|
|
165
|
+
for (const word of words(text)) {
|
|
166
|
+
if (isStopword(word))
|
|
167
|
+
continue;
|
|
168
|
+
for (const t of [word, stem(word)])
|
|
169
|
+
if (length(t) >= FTS_MIN_CHARS)
|
|
170
|
+
tokens.add(t);
|
|
171
|
+
}
|
|
172
|
+
tokens.add(text);
|
|
173
|
+
return [...tokens];
|
|
174
|
+
}
|
|
175
|
+
var ftsMatchExpression = (tokens) => tokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(" OR ");
|
|
176
|
+
var FTS_INSERT = "INSERT INTO skills_fts(uri, name, description, when_to_use, examples, aliases_ko, body_head) VALUES (?, ?, ?, ?, ?, ?, ?)";
|
|
177
|
+
var FTS_DELETE = "DELETE FROM skills_fts WHERE uri = ?";
|
|
178
|
+
var FTS_SEARCH = "SELECT uri, bm25(skills_fts, 0, 8, 2, 3, 5, 6, 1) AS score FROM skills_fts WHERE skills_fts MATCH ? ORDER BY score LIMIT ?";
|
|
179
|
+
var ftsFields = (e) => [e.name, e.description, e.whenToUse ?? "", e.examples.join("\n"), e.aliasesKo.join("\n"), e.bodyHead];
|
|
180
|
+
function ftsHits(rows, tokens, texts) {
|
|
181
|
+
return rows.map((row, i) => {
|
|
182
|
+
const text = texts.get(row.uri) ?? "";
|
|
183
|
+
return { uri: row.uri, rank: i + 1, bm25: row.score, matched: tokens.filter((t) => text.includes(t)) };
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// src/search/fts.ts
|
|
188
|
+
var FtsIndex = class {
|
|
189
|
+
constructor(db) {
|
|
190
|
+
this.db = db;
|
|
191
|
+
this.insertStmt = db.prepare(FTS_INSERT);
|
|
192
|
+
this.deleteStmt = db.prepare(FTS_DELETE);
|
|
193
|
+
this.searchStmt = db.prepare(FTS_SEARCH);
|
|
194
|
+
}
|
|
195
|
+
db;
|
|
196
|
+
insertStmt;
|
|
197
|
+
deleteStmt;
|
|
198
|
+
searchStmt;
|
|
199
|
+
texts = /* @__PURE__ */ new Map();
|
|
200
|
+
clear() {
|
|
201
|
+
this.db.exec("DELETE FROM skills_fts");
|
|
202
|
+
this.texts.clear();
|
|
203
|
+
}
|
|
204
|
+
remove(uris) {
|
|
205
|
+
for (const uri of uris) {
|
|
206
|
+
this.deleteStmt.run(uri);
|
|
207
|
+
this.texts.delete(uri);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
upsert(entries) {
|
|
211
|
+
for (const e of entries) {
|
|
212
|
+
this.deleteStmt.run(e.uri);
|
|
213
|
+
const fields = ftsFields(e);
|
|
214
|
+
this.insertStmt.run(e.uri, ...fields);
|
|
215
|
+
this.texts.set(e.uri, normalize(fields.join(" ")));
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
search(query, limit = 50) {
|
|
219
|
+
const tokens = ftsTokens(query);
|
|
220
|
+
if (!tokens.length) return [];
|
|
221
|
+
const rows = this.searchStmt.all(ftsMatchExpression(tokens), limit).map((row) => {
|
|
222
|
+
if (typeof row.uri !== "string" || typeof row.score !== "number") throw new TypeError("Invalid SQLite FTS row");
|
|
223
|
+
return { uri: row.uri, score: row.score };
|
|
224
|
+
});
|
|
225
|
+
return ftsHits(rows, tokens, this.texts);
|
|
226
|
+
}
|
|
227
|
+
};
|
|
228
|
+
|
|
229
|
+
// src/search/index.ts
|
|
230
|
+
function createSearchIndex(db) {
|
|
231
|
+
const entries = /* @__PURE__ */ new Map();
|
|
232
|
+
const fts = new FtsIndex(db);
|
|
233
|
+
let search = buildSearchCore([], fts);
|
|
234
|
+
const thresholdStmt = db.prepare("SELECT value FROM meta WHERE key = 'search_threshold'");
|
|
235
|
+
const threshold = () => {
|
|
236
|
+
const row = thresholdStmt.get();
|
|
237
|
+
const value = row?.value == null ? NaN : Number(row.value);
|
|
238
|
+
return Number.isFinite(value) ? value : DEFAULT_THRESHOLD;
|
|
239
|
+
};
|
|
240
|
+
const transaction = (fn) => {
|
|
241
|
+
db.exec("BEGIN");
|
|
242
|
+
try {
|
|
243
|
+
fn();
|
|
244
|
+
db.exec("COMMIT");
|
|
245
|
+
} catch (error) {
|
|
246
|
+
db.exec("ROLLBACK");
|
|
247
|
+
throw error;
|
|
248
|
+
}
|
|
249
|
+
};
|
|
250
|
+
return {
|
|
251
|
+
rebuild(list) {
|
|
252
|
+
transaction(() => {
|
|
253
|
+
fts.clear();
|
|
254
|
+
fts.upsert(list);
|
|
255
|
+
});
|
|
256
|
+
entries.clear();
|
|
257
|
+
for (const e of list) entries.set(e.uri, e);
|
|
258
|
+
search = buildSearchCore([...entries.values()], fts);
|
|
259
|
+
},
|
|
260
|
+
apply(delta) {
|
|
261
|
+
const upserts = [...delta.added, ...delta.changed];
|
|
262
|
+
transaction(() => {
|
|
263
|
+
fts.remove(delta.removed);
|
|
264
|
+
fts.upsert(upserts);
|
|
265
|
+
});
|
|
266
|
+
for (const uri of delta.removed) entries.delete(uri);
|
|
267
|
+
for (const e of upserts) entries.set(e.uri, e);
|
|
268
|
+
search = buildSearchCore([...entries.values()], fts);
|
|
269
|
+
},
|
|
270
|
+
search(query, opts) {
|
|
271
|
+
const started = performance.now();
|
|
272
|
+
const traceId = randomUUID();
|
|
273
|
+
return {
|
|
274
|
+
traceId,
|
|
275
|
+
query,
|
|
276
|
+
mode: opts.mode,
|
|
277
|
+
...search(query, opts, threshold()),
|
|
278
|
+
elapsedMs: performance.now() - started
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
};
|
|
74
282
|
}
|
|
283
|
+
export {
|
|
284
|
+
ABSTAIN_HINTS,
|
|
285
|
+
DEFAULT_THRESHOLD,
|
|
286
|
+
RRF_K,
|
|
287
|
+
SEARCH_TOOL_GUIDANCE,
|
|
288
|
+
calibrate,
|
|
289
|
+
createSearchIndex,
|
|
290
|
+
ftsTokens,
|
|
291
|
+
fuse,
|
|
292
|
+
normalize,
|
|
293
|
+
termCoverage,
|
|
294
|
+
terms
|
|
295
|
+
};
|
package/dist/search/lexical.js
CHANGED
|
@@ -1,69 +1,77 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
return
|
|
1
|
+
// ../shared/dist/search/fusion.js
|
|
2
|
+
var RRF_K = 60;
|
|
3
|
+
var MAX_RRF = 2 / (RRF_K + 1);
|
|
4
|
+
|
|
5
|
+
// ../shared/dist/search/lexical.js
|
|
6
|
+
var normalize = (value) => value.normalize("NFKC").toLowerCase().replace(/\s+/g, " ").trim();
|
|
7
|
+
var STOPWORDS = new Set("a an the and or to of for in on with from by is are be it this that do please use using when how me my can could would should not only then after before need want help \uD574\uC918 \uD574\uC8FC\uC138\uC694 \uC54C\uB824\uC918 \uC791\uC131\uD574\uC918 \uB9CC\uB4E4\uC5B4\uC918 \uD655\uC778\uD574\uC918 \uAC80\uD1A0\uD574\uC918 \uD544\uC694 \uC5C6\uC5B4 \uC788\uC2B5\uB2C8\uB2E4 \uD569\uB2C8\uB2E4 \uC791\uC5C5 \uC0AC\uC6A9 \uD1B5\uD574 \uB300\uD55C \uC704\uD55C \uAD00\uB828 \uD604\uC7AC \uB2E4\uC74C \uBA3C\uC800 \uC2E4\uC81C \uD558\uB098 \uAC83 \uC218 \uBC0F \uB610\uB294 \uD558\uACE0 \uADF8\uB9AC\uACE0 \uBC29\uBC95 \uC740 \uB294 \uC774 \uAC00 \uC744 \uB97C \uC758 \uB3C4 \uB85C \uC640 \uACFC \uC5D0".split(" "));
|
|
8
|
+
function stem(word) {
|
|
9
|
+
if (/^[가-힣]+$/.test(word) && word.length > 2) {
|
|
10
|
+
const stripped = word.replace(/(?:해주세요|해줘|하기|하게|하는|하면서|하려는|으로|에서|에게|에는|까지|부터|처럼|보다|은|는|을|를|이|가|과|와|의|도|로)$/, "");
|
|
11
|
+
return stripped.length >= 2 ? stripped : word;
|
|
12
|
+
}
|
|
13
|
+
return /^[a-z]{5,}s$/.test(word) && !word.endsWith("ss") ? word.slice(0, -1) : word;
|
|
12
14
|
}
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
return (normalize(text).match(/[\p{L}\p{N}]+/gu) ?? [])
|
|
16
|
-
.flatMap(word => word.split(/(?<=[가-힣])(?=[a-z0-9])|(?<=[a-z0-9])(?=[가-힣])/));
|
|
15
|
+
function words(text) {
|
|
16
|
+
return (normalize(text).match(/[\p{L}\p{N}]+/gu) ?? []).flatMap((word) => word.split(/(?<=[가-힣])(?=[a-z0-9])|(?<=[a-z0-9])(?=[가-힣])/));
|
|
17
17
|
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
18
|
+
var isStopword = (word) => STOPWORDS.has(word);
|
|
19
|
+
function terms(text) {
|
|
20
|
+
return [...new Set(words(text).filter((word) => !STOPWORDS.has(word)).map(stem).filter((word) => !STOPWORDS.has(word)))];
|
|
21
21
|
}
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
function keywordsOf(entry) {
|
|
23
|
+
return [...entry.examples, ...entry.aliasesKo, ...entry.whenToUse ? terms(entry.whenToUse) : []].map(normalize).filter(Boolean);
|
|
24
24
|
}
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
25
|
+
function buildLexical(entries) {
|
|
26
|
+
const frequency = /* @__PURE__ */ new Map();
|
|
27
|
+
const rows = entries.map((entry) => {
|
|
28
|
+
const name = normalize(entry.name), keywords = keywordsOf(entry);
|
|
29
|
+
const weights = /* @__PURE__ */ new Map();
|
|
30
|
+
for (const [field, weight] of [[entry.description, 1], [keywords.join(" "), 4], [name, 8]]) {
|
|
31
|
+
for (const term of terms(field))
|
|
32
|
+
weights.set(term, weight);
|
|
33
|
+
}
|
|
34
|
+
for (const term of weights.keys())
|
|
35
|
+
frequency.set(term, (frequency.get(term) ?? 0) + 1);
|
|
36
|
+
return { uri: entry.uri, name, keywords, nameTerms: new Set(terms(name)), weights };
|
|
37
|
+
});
|
|
38
|
+
const size = rows.length;
|
|
39
|
+
return {
|
|
40
|
+
search(query) {
|
|
41
|
+
const text = normalize(query);
|
|
42
|
+
if (!text)
|
|
43
|
+
return [];
|
|
44
|
+
const queryTerms = terms(text);
|
|
45
|
+
const hits = [];
|
|
46
|
+
for (const { uri, name, keywords, nameTerms, weights } of rows) {
|
|
47
|
+
const anchor = name === text ? 2 : keywords.includes(text) ? 1 : 0;
|
|
48
|
+
let score = anchor === 2 ? 1e3 : anchor === 1 ? 300 : 0;
|
|
49
|
+
let anchored = anchor > 0;
|
|
50
|
+
const matched = [];
|
|
51
|
+
for (const word of queryTerms) {
|
|
52
|
+
const weight = weights.get(word);
|
|
53
|
+
if (!weight)
|
|
54
|
+
continue;
|
|
55
|
+
matched.push(word);
|
|
56
|
+
anchored ||= nameTerms.has(word) || keywords.includes(word);
|
|
57
|
+
const rarity = Math.log(1 + (size + 0.5) / ((frequency.get(word) ?? 0) + 0.5));
|
|
58
|
+
score += weight * rarity;
|
|
33
59
|
}
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
if (!text)
|
|
43
|
-
return [];
|
|
44
|
-
const queryTerms = terms(text);
|
|
45
|
-
const hits = [];
|
|
46
|
-
for (const { uri, name, keywords, nameTerms, weights } of rows) {
|
|
47
|
-
const anchor = name === text ? 2 : keywords.includes(text) ? 1 : 0;
|
|
48
|
-
let score = anchor === 2 ? 1000 : anchor === 1 ? 300 : 0;
|
|
49
|
-
let anchored = anchor > 0;
|
|
50
|
-
const matched = [];
|
|
51
|
-
for (const word of queryTerms) {
|
|
52
|
-
const weight = weights.get(word);
|
|
53
|
-
if (!weight)
|
|
54
|
-
continue;
|
|
55
|
-
matched.push(word);
|
|
56
|
-
anchored ||= nameTerms.has(word) || keywords.includes(word);
|
|
57
|
-
const rarity = Math.log(1 + (size + .5) / ((frequency.get(word) ?? 0) + .5));
|
|
58
|
-
score += weight * rarity;
|
|
59
|
-
}
|
|
60
|
-
if (!score || (queryTerms.length > 1 && matched.length === 1 && !anchored))
|
|
61
|
-
continue;
|
|
62
|
-
// Prefer coverage without letting unmatched narrative details erase a result.
|
|
63
|
-
score *= 1 + matched.length / Math.max(queryTerms.length, 1);
|
|
64
|
-
hits.push({ uri, score, matched, anchor });
|
|
65
|
-
}
|
|
66
|
-
return hits.sort((a, b) => b.score - a.score || (a.uri < b.uri ? -1 : a.uri > b.uri ? 1 : 0));
|
|
67
|
-
},
|
|
68
|
-
};
|
|
60
|
+
if (!score || queryTerms.length > 1 && matched.length === 1 && !anchored)
|
|
61
|
+
continue;
|
|
62
|
+
score *= 1 + matched.length / Math.max(queryTerms.length, 1);
|
|
63
|
+
hits.push({ uri, score, matched, anchor });
|
|
64
|
+
}
|
|
65
|
+
return hits.sort((a, b) => b.score - a.score || (a.uri < b.uri ? -1 : a.uri > b.uri ? 1 : 0));
|
|
66
|
+
}
|
|
67
|
+
};
|
|
69
68
|
}
|
|
69
|
+
export {
|
|
70
|
+
buildLexical,
|
|
71
|
+
isStopword,
|
|
72
|
+
keywordsOf,
|
|
73
|
+
normalize,
|
|
74
|
+
stem,
|
|
75
|
+
terms,
|
|
76
|
+
words
|
|
77
|
+
};
|
package/dist/server.js
CHANGED
|
@@ -18,6 +18,11 @@ import { packageVersion } from "./mcp/tools.js";
|
|
|
18
18
|
import { createSearchIndex } from "./search/index.js";
|
|
19
19
|
import { openDb, selfCheck } from "./store/db.js";
|
|
20
20
|
import { createStore } from "./store/index.js";
|
|
21
|
+
import { startCloudSync } from "./sync/cloud-runner.js";
|
|
22
|
+
import { captureDaemonRuntime } from "./sync/cloud-runtime.js";
|
|
23
|
+
import { applySourceCommand } from "./sync/command-apply.js";
|
|
24
|
+
import { sourceCommandsEnabled } from "./sync/command-control.js";
|
|
25
|
+
import { readConnection } from "./sync/cloud-connection.js";
|
|
21
26
|
import { assertNodeVersion, assertPlatform, SkillarStartupError } from "./version.js";
|
|
22
27
|
export const HOST = "127.0.0.1";
|
|
23
28
|
export const LOG_MAX_BYTES = 5 * 1024 * 1024;
|
|
@@ -198,6 +203,13 @@ export async function startServer(config, deps = {}) {
|
|
|
198
203
|
});
|
|
199
204
|
const server = createServer(app);
|
|
200
205
|
boundPort = await listen(server, config.port);
|
|
206
|
+
const cloudSync = startCloudSync(skillarHome, dbPath(skillarHome), log, () => captureDaemonRuntime(db), deps.openStore === undefined ? (command, instanceId) => {
|
|
207
|
+
const connection = readConnection(skillarHome);
|
|
208
|
+
if (!sourceCommandsEnabled(skillarHome) || connection?.instanceId !== instanceId)
|
|
209
|
+
return null;
|
|
210
|
+
return applySourceCommand(db, ownStore, connection, command);
|
|
211
|
+
} : undefined);
|
|
212
|
+
cleanup.push(() => cloudSync.close());
|
|
201
213
|
let retentionTimer;
|
|
202
214
|
cleanup.push(() => watchdog.stop());
|
|
203
215
|
if (config.retentionDays !== null) {
|