singular-skill 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +15 -0
- package/.claude-plugin/plugin.json +10 -0
- package/LICENSE +21 -0
- package/README.md +399 -2
- package/bin/on-expand.js +277 -0
- package/bin/on-prompt.js +672 -0
- package/bin/on-subagent.js +127 -0
- package/bin/singular-cli.js +52 -0
- package/bin/singular.js +114 -0
- package/config.example.json +23 -0
- package/docs/security.md +110 -0
- package/docs/spec.md +209 -0
- package/docs/testing.md +298 -0
- package/hooks/hooks.json +38 -0
- package/lib/cache.js +93 -0
- package/lib/cli-core.js +862 -0
- package/lib/commands/add.js +339 -0
- package/lib/commands/find.js +195 -0
- package/lib/commands/mode.js +93 -0
- package/lib/commands/status.js +128 -0
- package/lib/config.js +407 -0
- package/lib/context.js +184 -0
- package/lib/git.js +173 -0
- package/lib/inspect.js +183 -0
- package/lib/log.js +67 -0
- package/lib/refs.js +101 -0
- package/lib/registry.js +1152 -0
- package/lib/score.js +416 -0
- package/lib/skill-index.js +684 -0
- package/lib/stack.js +401 -0
- package/package.json +50 -4
- package/skills/singular/SKILL.md +24 -0
- package/skills/singular-find/SKILL.md +18 -0
package/lib/registry.js
ADDED
|
@@ -0,0 +1,1152 @@
|
|
|
1
|
+
// lib/registry.js — skills.sh registry client for singular.
|
|
2
|
+
//
|
|
3
|
+
// searchSkills GET https://skills.sh/api/search?q=&limit= (anonymous JSON)
|
|
4
|
+
// fetchCatalog GET https://skills.sh/api/search?q=skill&owner=<o>&limit=200 (one request per allowlisted owner, cached 24 h)
|
|
5
|
+
// auditSkill GET https://skills.sh/api/v1/skills/audit/{source}/{skill}
|
|
6
|
+
// gateCandidates allowlist / install floor / blockedRefs / fork dedupe (pure)
|
|
7
|
+
// rankCandidates BM25-lite of the prompt (+ intent terms) against catalog names, + log10(installs) (pure)
|
|
8
|
+
// fetchSkillText npx -y skills@<v> use owner/repo#<sha>@skill (execFile, temp cwd, no shell; cached forever by pinned ref)
|
|
9
|
+
// sanitizeSkillText strip capability-carrying frontmatter keys (quoted / indented / flow forms too) and shell forms (pure)
|
|
10
|
+
// discover keywords -> catalogs (cached) + <= 2 tech-noun searches (cached) -> gate -> rank -> audit -> top N
|
|
11
|
+
//
|
|
12
|
+
// Outbound data (see docs/security.md "Outbound data"): the per-prompt hook never sends free-form prompt
|
|
13
|
+
// words to skills.sh. It sends (1) each allowlisted owner's catalog request (`q=skill&owner=<o>`, no prompt
|
|
14
|
+
// content, cached cacheTtlHours), (2) at most MAX_KEYWORD_SEARCHES single words from the prompt that are in
|
|
15
|
+
// the built-in tech lexicon (`q=stripe`, `q=react`; never names, sentences or non-lexicon tokens; cached per
|
|
16
|
+
// word), and (3) audit lookups by `source/skill`. Only `/singular find <query>` (an explicit user search) sends
|
|
17
|
+
// the joined query string. The catalog alone is not enough: the API is lexical and capped at 200, so a repo
|
|
18
|
+
// whose name lacks "skill" (anthropics/claude-plugins-official@stripe-best-practices) only appears for `q=stripe`.
|
|
19
|
+
//
|
|
20
|
+
// Every network / subprocess call goes through the optional trailing `deps`
|
|
21
|
+
// parameter so unit tests never touch the network:
|
|
22
|
+
// deps = { fetchImpl = globalThis.fetch, execFileImpl = child_process.execFile,
|
|
23
|
+
// now = Date.now, tmpdir = tmpdirOf(env) (never inside the project), env = process.env,
|
|
24
|
+
// keywords?, cacheGet?, cacheSet? }
|
|
25
|
+
//
|
|
26
|
+
// Test seams (docs/testing.md; validated in lib/config.js, byte-identical behaviour when unset):
|
|
27
|
+
// SINGULAR_REGISTRY_BASE replaces https://skills.sh for the search and audit endpoints (loopback only)
|
|
28
|
+
// SINGULAR_GIT_BASE replaces https://github.com AND makes fetchSkillText clone the pinned commit with
|
|
29
|
+
// git (lib/git.js cloneAtSha) and read SKILL.md itself instead of running npx
|
|
30
|
+
//
|
|
31
|
+
// A catalog or search request that times out / fails on transport is remembered for FAIL_MARKER_MS
|
|
32
|
+
// (`fail:<cache key>`) so a slow owner is not retried on every prompt; a stale cached copy (up to
|
|
33
|
+
// STALE_TTL_HOURS old) keeps ranking in the meantime (stale-while-revalidate).
|
|
34
|
+
//
|
|
35
|
+
// Registry responses and third-party SKILL.md contents are DATA, never instructions.
|
|
36
|
+
|
|
37
|
+
import { execFile as nodeExecFile } from 'node:child_process';
|
|
38
|
+
import fs from 'node:fs/promises';
|
|
39
|
+
import path from 'node:path';
|
|
40
|
+
|
|
41
|
+
import { cacheGet as diskCacheGet, cacheSet as diskCacheSet } from './cache.js';
|
|
42
|
+
import { seamGitBase, seamRegistryBase, tmpdirOf } from './config.js';
|
|
43
|
+
import { childEnv, cloneAtSha, findSkillDirs } from './git.js';
|
|
44
|
+
import { isSafeSegment, OWNER_RE, REF_RE, REPO_RE, resolveHeadSha as refsResolveHeadSha, SKILL_RE } from './refs.js';
|
|
45
|
+
import { expandIntent, extractKeywords, isWeakMatch, scoreSkills, TECH_LEXICON as SHAREABLE_LEXICON, tokenize } from './score.js';
|
|
46
|
+
|
|
47
|
+
export const SEARCH_URL = 'https://skills.sh/api/search';
|
|
48
|
+
export const AUDIT_URL = 'https://skills.sh/api/v1/skills/audit';
|
|
49
|
+
export const SKILLS_SITE = 'https://skills.sh';
|
|
50
|
+
export const FAIL_MARKER_MS = 5 * 60 * 1000; // a timed-out / failed catalog or search is not retried for 5 minutes
|
|
51
|
+
export const STALE_TTL_HOURS = 24 * 14; // a stale cached catalog still ranks while its owner is failing
|
|
52
|
+
|
|
53
|
+
/** Search / audit endpoints: production constants, or the loopback fixture server named by the test seam. */
|
|
54
|
+
export function endpoints(env = process.env) {
|
|
55
|
+
const base = seamRegistryBase(env);
|
|
56
|
+
if (!base) return { search: SEARCH_URL, audit: AUDIT_URL, base: SKILLS_SITE };
|
|
57
|
+
return { search: `${base}/api/search`, audit: `${base}/api/v1/skills/audit`, base };
|
|
58
|
+
}
|
|
59
|
+
export const MAX_SEARCH_LIMIT = 200; // server hard-caps at 200 (verified 2026-09-24)
|
|
60
|
+
export const SKILL_TEXT_CAP = 9000;
|
|
61
|
+
export const DEFAULT_CLI_VERSION = '1.5.26';
|
|
62
|
+
export const MIN_SEARCH_MS = 200; // discover(): skip a request when less than this remains
|
|
63
|
+
export const MIN_AUDIT_MS = 400; // discover(): skip audits when less than this remains
|
|
64
|
+
export const MAX_DISCOVER_QUERIES = 8;
|
|
65
|
+
export const MAX_CONCURRENT_REQUESTS = 2;
|
|
66
|
+
export const MAX_KEYWORD_SEARCHES = 2; // per prompt: single tech-lexicon words only (`q=stripe`), cached per word
|
|
67
|
+
export const CATALOG_QUERY = 'skill'; // `q=skill&owner=<o>` returns the owner's whole catalog (154 anthropics, 72 vercel-labs; 200 cap)
|
|
68
|
+
export const COOLDOWN_KEY = 'cooldown:skills.sh';
|
|
69
|
+
export const COOLDOWN_MS = 5 * 60 * 1000; // after an HTTP 429, no skills.sh request for 5 minutes
|
|
70
|
+
export const TEXT_CACHE_TTL_HOURS = 24 * 365 * 20; // a `#sha` ref is immutable: cache its text forever
|
|
71
|
+
export const MIN_CATALOG_MATCH_SCORE = 1.5;
|
|
72
|
+
|
|
73
|
+
// Frontmatter keys that grant capability when a skill is *invoked*. Inert in
|
|
74
|
+
// injected text, but stripped anyway (defense in depth).
|
|
75
|
+
export const STRIPPED_FRONTMATTER_KEYS = [
|
|
76
|
+
'allowed-tools',
|
|
77
|
+
'disallowed-tools',
|
|
78
|
+
'hooks',
|
|
79
|
+
'context',
|
|
80
|
+
'agent',
|
|
81
|
+
'shell',
|
|
82
|
+
'paths',
|
|
83
|
+
'model',
|
|
84
|
+
];
|
|
85
|
+
|
|
86
|
+
const DOMAIN_RE = /^[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?(?:\.[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?)+$/;
|
|
87
|
+
const SHA_RE = /^[0-9a-f]{40}$/;
|
|
88
|
+
const CLI_VERSION_RE = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/;
|
|
89
|
+
|
|
90
|
+
// ---------------------------------------------------------------------------
|
|
91
|
+
// deps
|
|
92
|
+
// ---------------------------------------------------------------------------
|
|
93
|
+
|
|
94
|
+
function withDeps(deps = {}) {
|
|
95
|
+
return {
|
|
96
|
+
fetchImpl: deps.fetchImpl ?? globalThis.fetch,
|
|
97
|
+
execFileImpl: deps.execFileImpl ?? nodeExecFile,
|
|
98
|
+
now: deps.now ?? Date.now,
|
|
99
|
+
tmpdir: deps.tmpdir ?? tmpdirOf(deps.env ?? process.env),
|
|
100
|
+
env: deps.env ?? process.env,
|
|
101
|
+
keywords: typeof deps.keywords === 'function' ? deps.keywords : null,
|
|
102
|
+
cacheGet: typeof deps.cacheGet === 'function' ? deps.cacheGet : null,
|
|
103
|
+
cacheSet: typeof deps.cacheSet === 'function' ? deps.cacheSet : null,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function getCache(d) {
|
|
108
|
+
if (d.cacheGet && d.cacheSet) return { get: d.cacheGet, set: d.cacheSet, impl: 'deps' };
|
|
109
|
+
return { get: (k, ttl) => diskCacheGet(k, ttl, d.env), set: (k, v) => diskCacheSet(k, v, d.env), impl: 'lib/cache.js' };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// ---------------------------------------------------------------------------
|
|
113
|
+
// small helpers
|
|
114
|
+
// ---------------------------------------------------------------------------
|
|
115
|
+
|
|
116
|
+
const lc = (s) => String(s ?? '').toLowerCase();
|
|
117
|
+
|
|
118
|
+
export function isValidSource(source) {
|
|
119
|
+
const s = String(source ?? '');
|
|
120
|
+
const parts = s.split('/');
|
|
121
|
+
if (parts.length === 2) return isSafeSegment(parts[0], OWNER_RE) && isSafeSegment(parts[1], REPO_RE);
|
|
122
|
+
if (parts.length === 1) return DOMAIN_RE.test(s); // well-known-domain sources (open.feishu.cn)
|
|
123
|
+
return false;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export function isValidSkillName(skill) {
|
|
127
|
+
return isSafeSegment(skill, SKILL_RE);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export function skillUrl(source, skill) {
|
|
131
|
+
return `${SKILLS_SITE}/${source}/${skill}`;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export function candidateRef(cand) {
|
|
135
|
+
return `${cand.source}@${cand.skill}`;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function byInstallsDesc(a, b) {
|
|
139
|
+
return b.installs - a.installs || a.source.localeCompare(b.source) || a.skill.localeCompare(b.skill);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function isCandidate(c) {
|
|
143
|
+
return (
|
|
144
|
+
c &&
|
|
145
|
+
typeof c === 'object' &&
|
|
146
|
+
typeof c.source === 'string' &&
|
|
147
|
+
typeof c.skill === 'string' &&
|
|
148
|
+
isValidSource(c.source) &&
|
|
149
|
+
isValidSkillName(c.skill)
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** Match `owner/repo` against allowlist entries of the form `owner` or `owner/repo` (case-insensitive). */
|
|
154
|
+
export function isAllowlisted(source, allowlist) {
|
|
155
|
+
const src = lc(source).trim();
|
|
156
|
+
const owner = src.split('/')[0];
|
|
157
|
+
if (!owner) return false;
|
|
158
|
+
for (const raw of Array.isArray(allowlist) ? allowlist : []) {
|
|
159
|
+
const entry = lc(raw).trim().replace(/^\/+|\/+$/g, '');
|
|
160
|
+
if (!entry) continue;
|
|
161
|
+
if (entry.includes('/')) {
|
|
162
|
+
if (entry === src) return true;
|
|
163
|
+
} else if (entry === owner) {
|
|
164
|
+
return true;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
return false;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Distinct owners named by an allowlist (`owner` or `owner/repo` entries), validated, in order. */
|
|
171
|
+
export function allowlistOwners(allowlist, max = 8) {
|
|
172
|
+
const owners = [];
|
|
173
|
+
for (const entry of Array.isArray(allowlist) ? allowlist : []) {
|
|
174
|
+
const owner = lc(entry).trim().replace(/^\/+/, '').split('/')[0];
|
|
175
|
+
if (isSafeSegment(owner, OWNER_RE) && !owners.includes(owner)) owners.push(owner);
|
|
176
|
+
if (owners.length >= max) break;
|
|
177
|
+
}
|
|
178
|
+
return owners;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function isBlocked(cand, blockedRefs) {
|
|
182
|
+
const src = lc(cand.source);
|
|
183
|
+
const owner = src.split('/')[0];
|
|
184
|
+
const full = `${src}@${lc(cand.skill)}`;
|
|
185
|
+
for (const raw of Array.isArray(blockedRefs) ? blockedRefs : []) {
|
|
186
|
+
const entry = lc(raw).trim().replace(/#[0-9a-f]{40}(?=@)/, '');
|
|
187
|
+
if (!entry) continue;
|
|
188
|
+
if (entry === full || entry === src || entry === owner) return true;
|
|
189
|
+
}
|
|
190
|
+
return false;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
async function fetchJson(url, { timeoutMs, fetchImpl }) {
|
|
194
|
+
const ac = new AbortController();
|
|
195
|
+
const ms = Math.max(1, Math.floor(Number(timeoutMs) || 0));
|
|
196
|
+
const timer = setTimeout(() => ac.abort(new Error(`timeout after ${ms}ms`)), ms);
|
|
197
|
+
timer.unref?.();
|
|
198
|
+
try {
|
|
199
|
+
const res = await fetchImpl(url, {
|
|
200
|
+
method: 'GET',
|
|
201
|
+
signal: ac.signal,
|
|
202
|
+
headers: { accept: 'application/json', 'user-agent': 'singular/0.2 (+node)' },
|
|
203
|
+
});
|
|
204
|
+
const status = Number(res?.status ?? 0);
|
|
205
|
+
const ok = res?.ok ?? (status >= 200 && status < 300);
|
|
206
|
+
let body = null;
|
|
207
|
+
try {
|
|
208
|
+
body = await res.json();
|
|
209
|
+
} catch {
|
|
210
|
+
body = null;
|
|
211
|
+
}
|
|
212
|
+
return { ok, status, body };
|
|
213
|
+
} finally {
|
|
214
|
+
clearTimeout(timer);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** Promise wrapper over an execFile-shaped function. Never rejects. */
|
|
219
|
+
function run(execFileImpl, file, args, options) {
|
|
220
|
+
return new Promise((resolve) => {
|
|
221
|
+
let settled = false;
|
|
222
|
+
const done = (error, stdout, stderr) => {
|
|
223
|
+
if (settled) return;
|
|
224
|
+
settled = true;
|
|
225
|
+
resolve({
|
|
226
|
+
error: error ?? null,
|
|
227
|
+
code: error ? (typeof error.code === 'number' ? error.code : null) : 0,
|
|
228
|
+
signal: error?.signal ?? null,
|
|
229
|
+
killed: Boolean(error?.killed),
|
|
230
|
+
stdout: String(stdout ?? ''),
|
|
231
|
+
stderr: String(stderr ?? ''),
|
|
232
|
+
});
|
|
233
|
+
};
|
|
234
|
+
let child;
|
|
235
|
+
try {
|
|
236
|
+
child = execFileImpl(file, args, options, done);
|
|
237
|
+
} catch (error) {
|
|
238
|
+
done(error, '', '');
|
|
239
|
+
return;
|
|
240
|
+
}
|
|
241
|
+
// The CLI must never wait on a TTY: close stdin (equivalent of `</dev/null`).
|
|
242
|
+
try {
|
|
243
|
+
child?.stdin?.end?.();
|
|
244
|
+
} catch {
|
|
245
|
+
/* ignore */
|
|
246
|
+
}
|
|
247
|
+
});
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
function describeRunFailure(what, r, timeoutMs) {
|
|
251
|
+
if (r.killed || r.signal) return `${what} timed out after ${timeoutMs}ms (signal ${r.signal ?? 'unknown'})`;
|
|
252
|
+
const err = r.stderr.trim() || r.error?.message || '';
|
|
253
|
+
const detail = err.split('\n').filter(Boolean).slice(-3).join(' | ').slice(0, 400);
|
|
254
|
+
const code = r.code ?? r.error?.code ?? 'unknown';
|
|
255
|
+
return `${what} failed (exit ${code})${detail ? `: ${detail}` : ''}`;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** Run `fn(item)` over `items` with at most `limit` in flight; resolves to results in order. */
|
|
259
|
+
async function mapLimit(items, limit, fn) {
|
|
260
|
+
const out = new Array(items.length);
|
|
261
|
+
let next = 0;
|
|
262
|
+
const worker = async () => {
|
|
263
|
+
while (next < items.length) {
|
|
264
|
+
const i = next++;
|
|
265
|
+
out[i] = await fn(items[i], i);
|
|
266
|
+
}
|
|
267
|
+
};
|
|
268
|
+
await Promise.all(Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, worker));
|
|
269
|
+
return out;
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
// ---------------------------------------------------------------------------
|
|
273
|
+
// searchSkills
|
|
274
|
+
// ---------------------------------------------------------------------------
|
|
275
|
+
|
|
276
|
+
/** Normalize raw /api/search entries into Candidate[] sorted by installs desc. */
|
|
277
|
+
export function normalizeSearchEntries(list) {
|
|
278
|
+
if (!Array.isArray(list)) return [];
|
|
279
|
+
const out = [];
|
|
280
|
+
for (const e of list) {
|
|
281
|
+
if (!e || typeof e !== 'object') continue;
|
|
282
|
+
const source = typeof e.source === 'string' ? e.source.trim() : '';
|
|
283
|
+
const skill =
|
|
284
|
+
typeof e.skillId === 'string' && e.skillId.trim()
|
|
285
|
+
? e.skillId.trim()
|
|
286
|
+
: typeof e.name === 'string'
|
|
287
|
+
? e.name.trim()
|
|
288
|
+
: '';
|
|
289
|
+
if (!source || !skill || !isValidSource(source) || !isValidSkillName(skill)) continue;
|
|
290
|
+
const n = Number(e.installs);
|
|
291
|
+
const installs = Number.isFinite(n) ? Math.max(0, Math.floor(n)) : 0;
|
|
292
|
+
out.push({ source, skill, installs, url: skillUrl(source, skill), isDuplicate: e.isDuplicate === true });
|
|
293
|
+
}
|
|
294
|
+
out.sort(byInstallsDesc);
|
|
295
|
+
return out;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/**
|
|
299
|
+
* Detailed search: distinguishes "no results" from "request failed" (so callers
|
|
300
|
+
* can avoid caching failures). Never throws. `status` carries the HTTP status (429 = rate limited).
|
|
301
|
+
*/
|
|
302
|
+
export async function searchSkillsRaw(query, opts = {}, deps = {}) {
|
|
303
|
+
const d = withDeps(deps);
|
|
304
|
+
const { limit = MAX_SEARCH_LIMIT, timeoutMs = 2500, owner = null } = opts;
|
|
305
|
+
const q = String(query ?? '').trim();
|
|
306
|
+
if (q.length < 2) return { ok: false, candidates: [], error: 'query must be at least 2 characters', status: 0 };
|
|
307
|
+
const n = Math.min(MAX_SEARCH_LIMIT, Math.max(1, Math.floor(Number(limit) || MAX_SEARCH_LIMIT)));
|
|
308
|
+
const params = new URLSearchParams({ q, limit: String(n) });
|
|
309
|
+
if (owner) {
|
|
310
|
+
if (!isSafeSegment(String(owner), OWNER_RE)) return { ok: false, candidates: [], error: 'invalid owner filter', status: 0 };
|
|
311
|
+
params.set('owner', String(owner));
|
|
312
|
+
}
|
|
313
|
+
const url = `${endpoints(d.env).search}?${params.toString()}`;
|
|
314
|
+
try {
|
|
315
|
+
const { ok, status, body } = await fetchJson(url, { timeoutMs, fetchImpl: d.fetchImpl });
|
|
316
|
+
if (!ok || !body || typeof body !== 'object') {
|
|
317
|
+
return { ok: false, candidates: [], error: `http ${status}${body?.error ? `: ${String(body.error).slice(0, 120)}` : ''}`, status };
|
|
318
|
+
}
|
|
319
|
+
const candidates = normalizeSearchEntries(body.skills);
|
|
320
|
+
return { ok: true, candidates, error: null, url, status, truncated: candidates.length >= n };
|
|
321
|
+
} catch (err) {
|
|
322
|
+
return { ok: false, candidates: [], error: String(err?.message ?? err), status: 0 };
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
/**
|
|
327
|
+
* searchSkills(query, {limit=200, timeoutMs}) -> Candidate[]
|
|
328
|
+
* Candidate = {source: "owner/repo", skill, installs, url, isDuplicate}
|
|
329
|
+
* Sorted by installs desc. Never throws; [] on any error.
|
|
330
|
+
*/
|
|
331
|
+
export async function searchSkills(query, opts = {}, deps = {}) {
|
|
332
|
+
const r = await searchSkillsRaw(query, opts, deps);
|
|
333
|
+
return r.candidates;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/**
|
|
337
|
+
* fetchCatalog(owner, {timeoutMs}) -> {ok, candidates, error, status, truncated}
|
|
338
|
+
* One request: every skill skills.sh lists for that owner (`q=skill&owner=<o>&limit=200`; the API's
|
|
339
|
+
* 200 cap truncates very large owners such as microsoft). No prompt content is sent.
|
|
340
|
+
*/
|
|
341
|
+
export async function fetchCatalog(owner, opts = {}, deps = {}) {
|
|
342
|
+
return searchSkillsRaw(CATALOG_QUERY, { limit: MAX_SEARCH_LIMIT, timeoutMs: opts.timeoutMs ?? 2500, owner }, deps);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
export function catalogCacheKey(owner) {
|
|
346
|
+
return `catalog:${lc(owner)}`;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/** Cooldown after a 429: {until} in the cache; `getCooldown` returns the `until` timestamp or 0. */
|
|
350
|
+
export async function getCooldown(cache, now = Date.now()) {
|
|
351
|
+
try {
|
|
352
|
+
const v = await cache.get(COOLDOWN_KEY, 1);
|
|
353
|
+
const until = Number(v?.until);
|
|
354
|
+
return Number.isFinite(until) && until > now ? until : 0;
|
|
355
|
+
} catch {
|
|
356
|
+
return 0;
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
export async function setCooldown(cache, now = Date.now(), ms = COOLDOWN_MS) {
|
|
361
|
+
const until = now + ms;
|
|
362
|
+
try {
|
|
363
|
+
await cache.set(COOLDOWN_KEY, { until, ts: now });
|
|
364
|
+
} catch {
|
|
365
|
+
/* ignore */
|
|
366
|
+
}
|
|
367
|
+
return until;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
// ---------------------------------------------------------------------------
|
|
371
|
+
// auditSkill
|
|
372
|
+
// ---------------------------------------------------------------------------
|
|
373
|
+
|
|
374
|
+
function normStatus(s) {
|
|
375
|
+
const v = lc(s).trim();
|
|
376
|
+
if (v === 'pass' || v === 'passed' || v === 'safe' || v === 'ok') return 'pass';
|
|
377
|
+
if (v === 'warn' || v === 'warning') return 'warn';
|
|
378
|
+
if (v === 'fail' || v === 'failed' || v === 'error') return 'fail';
|
|
379
|
+
return 'unknown';
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/** Aggregate 3-5 provider verdicts: any fail (or CRITICAL risk) -> fail; any warn -> warn; else pass; none -> unknown. */
|
|
383
|
+
export function summarizeAudits(audits) {
|
|
384
|
+
if (!Array.isArray(audits) || audits.length === 0) return { status: 'unknown', providers: [] };
|
|
385
|
+
const providers = audits
|
|
386
|
+
.filter((a) => a && typeof a === 'object')
|
|
387
|
+
.map((a) => ({
|
|
388
|
+
provider: String(a.provider ?? a.slug ?? 'unknown'),
|
|
389
|
+
status: normStatus(a.status),
|
|
390
|
+
riskLevel: a.riskLevel != null ? String(a.riskLevel).toUpperCase() : null,
|
|
391
|
+
summary: typeof a.summary === 'string' ? a.summary.slice(0, 200) : '',
|
|
392
|
+
}));
|
|
393
|
+
const counts = { pass: 0, warn: 0, fail: 0, unknown: 0 };
|
|
394
|
+
for (const p of providers) {
|
|
395
|
+
if (p.riskLevel === 'CRITICAL') counts.fail++;
|
|
396
|
+
else counts[p.status]++;
|
|
397
|
+
}
|
|
398
|
+
let status = 'unknown';
|
|
399
|
+
if (counts.fail > 0) status = 'fail';
|
|
400
|
+
else if (counts.warn > 0) status = 'warn';
|
|
401
|
+
else if (counts.pass > 0) status = 'pass';
|
|
402
|
+
return { status, providers };
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
/**
|
|
406
|
+
* auditSkill(source, skill, {timeoutMs}) -> {status: "pass"|"warn"|"fail"|"unknown", raw, providers}
|
|
407
|
+
* 404 (no audits yet) and any transport error -> "unknown". Never throws.
|
|
408
|
+
*/
|
|
409
|
+
export async function auditSkill(source, skill, opts = {}, deps = {}) {
|
|
410
|
+
const d = withDeps(deps);
|
|
411
|
+
const { timeoutMs = 2500 } = opts;
|
|
412
|
+
const src = String(source ?? '').trim();
|
|
413
|
+
const sk = String(skill ?? '').trim();
|
|
414
|
+
if (!isValidSource(src) || !isValidSkillName(sk)) {
|
|
415
|
+
return { status: 'unknown', raw: { error: 'invalid_ref' }, providers: [] };
|
|
416
|
+
}
|
|
417
|
+
const url = `${endpoints(d.env).audit}/${src.split('/').map(encodeURIComponent).join('/')}/${encodeURIComponent(sk)}`;
|
|
418
|
+
try {
|
|
419
|
+
const { ok, status, body } = await fetchJson(url, { timeoutMs, fetchImpl: d.fetchImpl });
|
|
420
|
+
if (status === 404) return { status: 'unknown', raw: body ?? { httpStatus: 404 }, providers: [] };
|
|
421
|
+
if (!ok || !body || typeof body !== 'object') {
|
|
422
|
+
return { status: 'unknown', raw: { httpStatus: status, error: body?.error ?? null, transient: true, rateLimited: status === 429 }, providers: [] };
|
|
423
|
+
}
|
|
424
|
+
const agg = summarizeAudits(body.audits);
|
|
425
|
+
return { status: agg.status, raw: body, providers: agg.providers };
|
|
426
|
+
} catch (err) {
|
|
427
|
+
return { status: 'unknown', raw: { error: String(err?.message ?? err), transient: true }, providers: [] };
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
// ---------------------------------------------------------------------------
|
|
432
|
+
// gateCandidates
|
|
433
|
+
// ---------------------------------------------------------------------------
|
|
434
|
+
|
|
435
|
+
/** Merge candidates from several searches: dedupe by source@skill, keep max installs. */
|
|
436
|
+
export function mergeCandidates(lists) {
|
|
437
|
+
const map = new Map();
|
|
438
|
+
for (const list of lists) {
|
|
439
|
+
for (const c of Array.isArray(list) ? list : []) {
|
|
440
|
+
if (!isCandidate(c)) continue;
|
|
441
|
+
const key = lc(candidateRef(c));
|
|
442
|
+
const prev = map.get(key);
|
|
443
|
+
if (!prev) {
|
|
444
|
+
map.set(key, {
|
|
445
|
+
source: c.source,
|
|
446
|
+
skill: c.skill,
|
|
447
|
+
installs: Number(c.installs) || 0,
|
|
448
|
+
url: c.url ?? skillUrl(c.source, c.skill),
|
|
449
|
+
isDuplicate: c.isDuplicate === true,
|
|
450
|
+
});
|
|
451
|
+
} else {
|
|
452
|
+
prev.installs = Math.max(prev.installs, Number(c.installs) || 0);
|
|
453
|
+
prev.isDuplicate = prev.isDuplicate || c.isDuplicate === true;
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
return [...map.values()].sort(byInstallsDesc);
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
/**
|
|
461
|
+
* gateCandidates(cands, config, extra={}) -> {accepted: Candidate[], rejected: {cand, reason}[]}
|
|
462
|
+
* Applies: blockedRefs, already-installed (extra.installedNames), owner allowlist
|
|
463
|
+
* (owner or owner/repo, case-insensitive), install floor, fork dedupe (same skill
|
|
464
|
+
* name: allowlisted wins; among allowlisted, highest installs wins).
|
|
465
|
+
* `isDuplicate: true` from the API is NOT a rejection reason (the canonical
|
|
466
|
+
* entry is often the one flagged).
|
|
467
|
+
*/
|
|
468
|
+
export function gateCandidates(cands, config = {}, extra = {}) {
|
|
469
|
+
const allowlist = extra.allowlist ?? config.allowlist ?? [];
|
|
470
|
+
const installFloor = Number(extra.installFloor ?? config.installFloor ?? 0) || 0;
|
|
471
|
+
const blockedRefs = extra.blockedRefs ?? config.blockedRefs ?? [];
|
|
472
|
+
const installed = new Set((Array.isArray(extra.installedNames) ? extra.installedNames : []).map(lc));
|
|
473
|
+
|
|
474
|
+
const rejected = [];
|
|
475
|
+
const prelim = [];
|
|
476
|
+
for (const cand of mergeCandidates([cands])) {
|
|
477
|
+
const owner = cand.source.split('/')[0];
|
|
478
|
+
if (isBlocked(cand, blockedRefs)) {
|
|
479
|
+
rejected.push({ cand, reason: 'blocked by blockedRefs' });
|
|
480
|
+
continue;
|
|
481
|
+
}
|
|
482
|
+
if (installed.has(lc(cand.skill))) {
|
|
483
|
+
rejected.push({ cand, reason: `already installed locally (${cand.skill})` });
|
|
484
|
+
continue;
|
|
485
|
+
}
|
|
486
|
+
if (!isAllowlisted(cand.source, allowlist)) {
|
|
487
|
+
prelim.push({ cand, allow: false, owner });
|
|
488
|
+
continue;
|
|
489
|
+
}
|
|
490
|
+
if (cand.installs < installFloor) {
|
|
491
|
+
rejected.push({ cand, reason: `installs ${cand.installs} < floor ${installFloor}` });
|
|
492
|
+
continue;
|
|
493
|
+
}
|
|
494
|
+
prelim.push({ cand, allow: true, owner });
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
const accepted = [];
|
|
498
|
+
const bestByName = new Map();
|
|
499
|
+
for (const { cand, allow } of prelim) {
|
|
500
|
+
if (!allow) continue;
|
|
501
|
+
const key = lc(cand.skill);
|
|
502
|
+
const best = bestByName.get(key);
|
|
503
|
+
if (!best) {
|
|
504
|
+
bestByName.set(key, cand);
|
|
505
|
+
accepted.push(cand);
|
|
506
|
+
} else {
|
|
507
|
+
rejected.push({ cand, reason: `duplicate skill name; kept ${candidateRef(best)} (${best.installs} installs)` });
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
for (const { cand, allow, owner } of prelim) {
|
|
511
|
+
if (allow) continue;
|
|
512
|
+
const best = bestByName.get(lc(cand.skill));
|
|
513
|
+
rejected.push({
|
|
514
|
+
cand,
|
|
515
|
+
reason: best
|
|
516
|
+
? `fork of ${candidateRef(best)} (owner ${owner} not allowlisted)`
|
|
517
|
+
: `owner ${owner} not allowlisted`,
|
|
518
|
+
});
|
|
519
|
+
}
|
|
520
|
+
rejected.sort((a, b) => byInstallsDesc(a.cand, b.cand));
|
|
521
|
+
return { accepted, rejected };
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
// ---------------------------------------------------------------------------
|
|
525
|
+
// refs / sha resolution
|
|
526
|
+
// ---------------------------------------------------------------------------
|
|
527
|
+
|
|
528
|
+
/** Local ref parser (same grammar as lib/refs.js): owner/repo@skill | owner/repo#<40-hex>@skill */
|
|
529
|
+
export function parseRefLocal(str) {
|
|
530
|
+
const m = /^([A-Za-z0-9][A-Za-z0-9-]*)\/([A-Za-z0-9][A-Za-z0-9._-]*)(?:#([0-9a-f]{40}))?@([A-Za-z0-9][A-Za-z0-9._-]*)$/.exec(String(str ?? '').trim());
|
|
531
|
+
if (!m) return null;
|
|
532
|
+
if (!isSafeSegment(m[1], OWNER_RE) || !isSafeSegment(m[2], REPO_RE) || !isSafeSegment(m[4], SKILL_RE)) return null;
|
|
533
|
+
return { owner: m[1], repo: m[2], sha: m[3] ?? null, skill: m[4] };
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
function normalizeRef(ref) {
|
|
537
|
+
if (typeof ref === 'string') return parseRefLocal(ref);
|
|
538
|
+
if (!ref || typeof ref !== 'object') return null;
|
|
539
|
+
let owner = ref.owner;
|
|
540
|
+
let repo = ref.repo;
|
|
541
|
+
if ((!owner || !repo) && typeof ref.source === 'string') {
|
|
542
|
+
const parts = ref.source.split('/');
|
|
543
|
+
if (parts.length === 2) [owner, repo] = parts;
|
|
544
|
+
}
|
|
545
|
+
const skill = ref.skill;
|
|
546
|
+
const sha = ref.sha == null || ref.sha === '' ? null : String(ref.sha).toLowerCase();
|
|
547
|
+
if (!isSafeSegment(owner, OWNER_RE) || !isSafeSegment(repo, REPO_RE) || !isSafeSegment(skill, SKILL_RE)) return null;
|
|
548
|
+
if (sha !== null && !SHA_RE.test(sha)) return null;
|
|
549
|
+
return { owner: String(owner), repo: String(repo), sha, skill: String(skill) };
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
/** resolveHeadSha(owner, repo, {timeoutMs}) via `git ls-remote <base>/<o>/<r> HEAD` (execFile, no shell); see lib/refs.js. */
|
|
553
|
+
export async function resolveHeadSha(owner, repo, opts = {}, deps = {}) {
|
|
554
|
+
const d = withDeps(deps);
|
|
555
|
+
return refsResolveHeadSha(owner, repo, opts, { execFileImpl: d.execFileImpl, env: d.env });
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
// ---------------------------------------------------------------------------
|
|
559
|
+
// fetchSkillText
|
|
560
|
+
// ---------------------------------------------------------------------------
|
|
561
|
+
|
|
562
|
+
/**
|
|
563
|
+
* Strip the `skills use` wrapper prose. Verified markers (skills@1.5.26):
|
|
564
|
+
* PREFIX ends with "<SKILL.md>\n", SUFFIX is "\n</SKILL.md>\n".
|
|
565
|
+
* Returns the raw SKILL.md text, or null when the markers are absent.
|
|
566
|
+
*/
|
|
567
|
+
export function stripCliWrapper(stdout) {
|
|
568
|
+
const s = String(stdout ?? '');
|
|
569
|
+
const open = '<SKILL.md>\n';
|
|
570
|
+
const close = '\n</SKILL.md>';
|
|
571
|
+
const a = s.indexOf(open);
|
|
572
|
+
if (a < 0) return null;
|
|
573
|
+
const b = s.lastIndexOf(close);
|
|
574
|
+
if (b < a + open.length) return null;
|
|
575
|
+
return s.slice(a + open.length, b);
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
export function textCacheKey(pinnedRef) {
|
|
579
|
+
return `text:${pinnedRef}`;
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
/**
|
|
583
|
+
* fetchSkillText(ref, {cliVersion, timeoutMs, cache}) -> {text, sha, ref, cached}
|
|
584
|
+
* ref: "owner/repo@skill" | "owner/repo#<sha>@skill" | {owner, repo, sha|null, skill}
|
|
585
|
+
* Runs `npx -y skills@<cliVersion> use owner/repo#<sha>@skill` with DISABLE_TELEMETRY=1,
|
|
586
|
+
* cwd and TMPDIR = a fresh temp dir (removed afterwards; the CLI leaks
|
|
587
|
+
* $TMPDIR/skills-use-* otherwise), execFile with an args array (no shell),
|
|
588
|
+
* timeouts on everything. If ref.sha is null the HEAD sha is resolved first.
|
|
589
|
+
* With `cache: true` the raw text is cached forever under `text:<pinned ref>` (a
|
|
590
|
+
* `#sha` ref is immutable), so any ref is fetched at most once per machine.
|
|
591
|
+
* Throws on invalid ref, bad sha, CLI failure, or missing wrapper markers.
|
|
592
|
+
*/
|
|
593
|
+
export async function fetchSkillText(ref, opts = {}, deps = {}) {
|
|
594
|
+
const d = withDeps(deps);
|
|
595
|
+
const { cliVersion = DEFAULT_CLI_VERSION, timeoutMs = 20000, cache: useCache = false } = opts;
|
|
596
|
+
if (!CLI_VERSION_RE.test(String(cliVersion))) throw new Error(`invalid skills CLI version: ${String(cliVersion)}`);
|
|
597
|
+
const parsed = normalizeRef(ref);
|
|
598
|
+
if (!parsed) throw new Error(`invalid ref: ${typeof ref === 'string' ? ref : JSON.stringify(ref)}`);
|
|
599
|
+
|
|
600
|
+
let sha = parsed.sha;
|
|
601
|
+
if (!sha) sha = await resolveHeadSha(parsed.owner, parsed.repo, { timeoutMs: Math.min(timeoutMs, 8000) }, d);
|
|
602
|
+
if (!SHA_RE.test(String(sha))) throw new Error(`resolved sha is not a 40-hex commit sha: ${String(sha)}`);
|
|
603
|
+
|
|
604
|
+
const full = `${parsed.owner}/${parsed.repo}#${sha}@${parsed.skill}`;
|
|
605
|
+
if (!REF_RE.test(full)) throw new Error(`ref failed validation: ${full}`);
|
|
606
|
+
|
|
607
|
+
const cache = useCache ? getCache(d) : null;
|
|
608
|
+
if (cache) {
|
|
609
|
+
try {
|
|
610
|
+
const hit = await cache.get(textCacheKey(full), TEXT_CACHE_TTL_HOURS);
|
|
611
|
+
if (hit && typeof hit === 'object' && typeof hit.text === 'string') return { text: hit.text, sha, ref: full, cached: true };
|
|
612
|
+
} catch {
|
|
613
|
+
/* cache miss */
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
const base = path.resolve(String(d.tmpdir));
|
|
618
|
+
await fs.mkdir(base, { recursive: true });
|
|
619
|
+
const work = await fs.mkdtemp(path.join(base, 'singular-use-'));
|
|
620
|
+
try {
|
|
621
|
+
let text;
|
|
622
|
+
if (seamGitBase(d.env)) {
|
|
623
|
+
// Test seam: no npx; clone the pinned commit from the local git base and read SKILL.md ourselves.
|
|
624
|
+
text = await fetchSkillTextViaGit(parsed, sha, { work, timeoutMs, execFileImpl: d.execFileImpl, env: d.env });
|
|
625
|
+
} else {
|
|
626
|
+
// Allowlisted environment only (lib/git.js childEnv): no inherited npm_config_* (registry, cache,
|
|
627
|
+
// userconfig), NODE_OPTIONS / NODE_PATH, GIT_* or CA overrides reach npx; cwd and TMPDIR are the fresh work
|
|
628
|
+
// dir, which hardenEnv keeps outside the project so no repository .npmrc is picked up on the walk up.
|
|
629
|
+
const env = childEnv(d.env, {
|
|
630
|
+
DISABLE_TELEMETRY: '1',
|
|
631
|
+
TMPDIR: work,
|
|
632
|
+
NO_UPDATE_NOTIFIER: '1',
|
|
633
|
+
});
|
|
634
|
+
const args = ['-y', `skills@${cliVersion}`, 'use', full];
|
|
635
|
+
const r = await run(d.execFileImpl, 'npx', args, {
|
|
636
|
+
cwd: work,
|
|
637
|
+
env,
|
|
638
|
+
timeout: timeoutMs,
|
|
639
|
+
killSignal: 'SIGKILL',
|
|
640
|
+
maxBuffer: 4 * 1024 * 1024,
|
|
641
|
+
windowsHide: true,
|
|
642
|
+
});
|
|
643
|
+
if (r.error || r.code !== 0) throw new Error(describeRunFailure(`skills use ${full}`, r, timeoutMs));
|
|
644
|
+
text = stripCliWrapper(r.stdout);
|
|
645
|
+
if (text === null) throw new Error(`unexpected skills CLI output for ${full}: SKILL.md markers not found`);
|
|
646
|
+
}
|
|
647
|
+
if (cache) {
|
|
648
|
+
try {
|
|
649
|
+
await cache.set(textCacheKey(full), { text, sha, ref: full, ts: d.now() });
|
|
650
|
+
} catch {
|
|
651
|
+
/* ignore */
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
return { text, sha, ref: full, cached: false };
|
|
655
|
+
} finally {
|
|
656
|
+
if (work.startsWith(base + path.sep)) {
|
|
657
|
+
await fs.rm(work, { recursive: true, force: true }).catch(() => {});
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
/**
|
|
663
|
+
* Git-based text fetch (SINGULAR_GIT_BASE seam only): clone `owner/repo` at `sha` into <work>/repo,
|
|
664
|
+
* locate the skill by an exact directory or frontmatter name (same rule as /singular add) and return its
|
|
665
|
+
* SKILL.md. Throws with the same message shapes as the npx path.
|
|
666
|
+
*/
|
|
667
|
+
async function fetchSkillTextViaGit(parsed, sha, { work, timeoutMs, execFileImpl, env }) {
|
|
668
|
+
const dest = path.join(work, 'repo');
|
|
669
|
+
await fs.mkdir(dest, { recursive: true });
|
|
670
|
+
const cloned = await cloneAtSha({ owner: parsed.owner, repo: parsed.repo, sha, dest, timeoutMs, execFileImpl, env, tmpdir: work });
|
|
671
|
+
if (!cloned.ok) throw new Error(`git fetch ${parsed.owner}/${parsed.repo}#${sha}@${parsed.skill}: ${cloned.message}`);
|
|
672
|
+
const all = findSkillDirs(dest);
|
|
673
|
+
const hit = all.filter((s) => path.basename(s.dir) === parsed.skill || s.name === parsed.skill);
|
|
674
|
+
if (hit.length !== 1) {
|
|
675
|
+
const names = [...new Set(all.map((s) => path.basename(s.dir)))].slice(0, 30).join(', ') || 'none';
|
|
676
|
+
throw new Error(hit.length === 0 ? `No matching skill found for: ${parsed.skill} (found: ${names})` : `ambiguous skill name ${parsed.skill}: ${hit.map((h) => h.rel).join(', ')}`);
|
|
677
|
+
}
|
|
678
|
+
return fs.readFile(path.join(hit[0].dir, 'SKILL.md'), 'utf8');
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
// ---------------------------------------------------------------------------
|
|
682
|
+
// sanitizeSkillText
|
|
683
|
+
// ---------------------------------------------------------------------------
|
|
684
|
+
|
|
685
|
+
// Any indentation, optional matching quotes around the key: Claude Code honours `"allowed-tools":` and
|
|
686
|
+
// ` hooks:` (verified 2.1.282), so the sanitizer must too. Fail closed on anything the minimal parser
|
|
687
|
+
// cannot interpret (flow mappings `{`/`[` at line start are dropped).
|
|
688
|
+
export const FM_KEY_RE = /^(\s*)(["']?)([A-Za-z0-9_.-]+)\2\s*:(?:\s|$)/;
|
|
689
|
+
const FM_FLOW_RE = /^\s*[{[]/;
|
|
690
|
+
const FENCE_BANG_RE = /^\s*(`{3,}|~{3,})\s*!/;
|
|
691
|
+
const FENCE_CLOSE_RE = /^\s*(`{3,}|~{3,})\s*$/;
|
|
692
|
+
const AT_REF_RE = /^\s*(?:[-*+]\s+)?@[^\s@]/;
|
|
693
|
+
const ANSI_RE = /\x1b\[[0-9;?]*[ -/]*[@-~]/g;
|
|
694
|
+
|
|
695
|
+
function splitFrontmatter(lines) {
|
|
696
|
+
if (lines.length === 0 || lines[0].trim() !== '---') return null;
|
|
697
|
+
for (let i = 1; i < lines.length; i++) {
|
|
698
|
+
if (lines[i].trim() === '---') return { fm: lines.slice(1, i), body: lines.slice(i + 1) };
|
|
699
|
+
}
|
|
700
|
+
return null;
|
|
701
|
+
}
|
|
702
|
+
|
|
703
|
+
function indentOf(line) {
|
|
704
|
+
return /^\s*/.exec(line)[0].length;
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
/** Index of the first line after the block that `key` at `keyIndent` on line `i` owns (nested lines, list items, blanks). */
|
|
708
|
+
function skipNested(lines, i, keyIndent) {
|
|
709
|
+
let j = i + 1;
|
|
710
|
+
while (j < lines.length) {
|
|
711
|
+
const l = lines[j];
|
|
712
|
+
if (l.trim() === '') {
|
|
713
|
+
j++;
|
|
714
|
+
continue;
|
|
715
|
+
}
|
|
716
|
+
const ind = indentOf(l);
|
|
717
|
+
if (ind > keyIndent || (ind === keyIndent && /^\s*-\s/.test(l))) {
|
|
718
|
+
j++;
|
|
719
|
+
continue;
|
|
720
|
+
}
|
|
721
|
+
break;
|
|
722
|
+
}
|
|
723
|
+
return j;
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
/**
|
|
727
|
+
* sanitizeSkillText(text) -> {text, stripped: string[]}
|
|
728
|
+
* - removes frontmatter keys allowed-tools, disallowed-tools, hooks, context, agent, shell, paths, model
|
|
729
|
+
* (with nested lines) in plain, quoted (`"hooks":`) and indented (` hooks:`) forms
|
|
730
|
+
* - drops flow-mapping lines (`{` / `[` at line start) inside the frontmatter
|
|
731
|
+
* - drops lines containing !` and fenced ```! / ~~~! blocks (inclusive)
|
|
732
|
+
* - drops @-file references at line start (also `- @path` list items)
|
|
733
|
+
* - strips ANSI escapes / NUL, normalizes CRLF
|
|
734
|
+
* - caps at 9,000 chars
|
|
735
|
+
*/
|
|
736
|
+
export function sanitizeSkillText(text) {
|
|
737
|
+
const stripped = [];
|
|
738
|
+
let s = String(text ?? '').replace(/^/, '');
|
|
739
|
+
const noCtl = s.replace(ANSI_RE, '').replace(/\0/g, '');
|
|
740
|
+
if (noCtl !== s) stripped.push('control/ANSI escape sequences');
|
|
741
|
+
s = noCtl.replace(/\r\n?/g, '\n');
|
|
742
|
+
|
|
743
|
+
const lines = s.split('\n');
|
|
744
|
+
const dropKeys = new Set(STRIPPED_FRONTMATTER_KEYS);
|
|
745
|
+
const outLines = [];
|
|
746
|
+
|
|
747
|
+
const fmSplit = splitFrontmatter(lines);
|
|
748
|
+
let bodyLines = lines;
|
|
749
|
+
if (fmSplit) {
|
|
750
|
+
const kept = [];
|
|
751
|
+
let i = 0;
|
|
752
|
+
while (i < fmSplit.fm.length) {
|
|
753
|
+
const line = fmSplit.fm[i];
|
|
754
|
+
const m = FM_KEY_RE.exec(line);
|
|
755
|
+
if (m && dropKeys.has(m[3].toLowerCase())) {
|
|
756
|
+
const key = m[3].toLowerCase();
|
|
757
|
+
i = skipNested(fmSplit.fm, i, m[1].length);
|
|
758
|
+
stripped.push(`frontmatter: ${key}`);
|
|
759
|
+
continue;
|
|
760
|
+
}
|
|
761
|
+
if (FM_FLOW_RE.test(line)) {
|
|
762
|
+
stripped.push(`frontmatter: flow mapping line ${i + 1} removed`);
|
|
763
|
+
i++;
|
|
764
|
+
continue;
|
|
765
|
+
}
|
|
766
|
+
if (line.includes('!`')) {
|
|
767
|
+
stripped.push(`frontmatter: inline shell (!\`) removed`);
|
|
768
|
+
i++;
|
|
769
|
+
continue;
|
|
770
|
+
}
|
|
771
|
+
kept.push(line);
|
|
772
|
+
i++;
|
|
773
|
+
}
|
|
774
|
+
const meaningful = kept.some((l) => l.trim() !== '');
|
|
775
|
+
if (meaningful) outLines.push('---', ...kept, '---');
|
|
776
|
+
else stripped.push('frontmatter: emptied');
|
|
777
|
+
bodyLines = fmSplit.body;
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
let inBangFence = null; // {start, marker}
|
|
781
|
+
let fenceLines = 0;
|
|
782
|
+
for (let i = 0; i < bodyLines.length; i++) {
|
|
783
|
+
const line = bodyLines[i];
|
|
784
|
+
const lineNo = i + 1;
|
|
785
|
+
if (inBangFence) {
|
|
786
|
+
fenceLines++;
|
|
787
|
+
const close = FENCE_CLOSE_RE.exec(line);
|
|
788
|
+
if (close && close[1][0] === inBangFence.marker) {
|
|
789
|
+
stripped.push(`fenced shell block (\`\`\`!) lines ${inBangFence.start}-${lineNo} (${fenceLines} lines)`);
|
|
790
|
+
inBangFence = null;
|
|
791
|
+
}
|
|
792
|
+
continue;
|
|
793
|
+
}
|
|
794
|
+
const open = FENCE_BANG_RE.exec(line);
|
|
795
|
+
if (open) {
|
|
796
|
+
inBangFence = { start: lineNo, marker: open[1][0] };
|
|
797
|
+
fenceLines = 1;
|
|
798
|
+
continue;
|
|
799
|
+
}
|
|
800
|
+
if (line.includes('!`')) {
|
|
801
|
+
stripped.push(`inline shell (!\`) line ${lineNo}`);
|
|
802
|
+
continue;
|
|
803
|
+
}
|
|
804
|
+
if (AT_REF_RE.test(line)) {
|
|
805
|
+
stripped.push(`@-file reference line ${lineNo}: ${line.trim().slice(0, 60)}`);
|
|
806
|
+
continue;
|
|
807
|
+
}
|
|
808
|
+
outLines.push(line);
|
|
809
|
+
}
|
|
810
|
+
if (inBangFence) stripped.push(`unterminated fenced shell block (\`\`\`!) from line ${inBangFence.start} (${fenceLines} lines)`);
|
|
811
|
+
|
|
812
|
+
let out = outLines.join('\n');
|
|
813
|
+
if (out.length > SKILL_TEXT_CAP) {
|
|
814
|
+
const marker = '\n[singular: truncated]';
|
|
815
|
+
const cut = out.length - (SKILL_TEXT_CAP - marker.length);
|
|
816
|
+
out = out.slice(0, SKILL_TEXT_CAP - marker.length) + marker;
|
|
817
|
+
stripped.push(`truncated ${cut} chars to ${SKILL_TEXT_CAP}`);
|
|
818
|
+
}
|
|
819
|
+
return { text: out, stripped };
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
// ---------------------------------------------------------------------------
|
|
823
|
+
// discover
|
|
824
|
+
// ---------------------------------------------------------------------------
|
|
825
|
+
|
|
826
|
+
const STOPWORDS = new Set(
|
|
827
|
+
(
|
|
828
|
+
'a an the and or but if then else of to in on at for with without from by as is are was were be been being ' +
|
|
829
|
+
'it its this that these those i me my we our you your he she they them their what which who whom how when ' +
|
|
830
|
+
'where why can could should would will shall may might must do does did done have has had having not no nor ' +
|
|
831
|
+
'so than too very just also into over under again further once here there all any both each few more most ' +
|
|
832
|
+
'other some such only own same now please help want need make build create add write fix use using get set ' +
|
|
833
|
+
'let like new code file files project app thing things something way work working'
|
|
834
|
+
).split(' '),
|
|
835
|
+
);
|
|
836
|
+
const TECH_LEXICON = new Set(
|
|
837
|
+
(
|
|
838
|
+
'react nextjs next tailwind svelte vue nuxt astro remix angular django fastapi flask postgres postgresql ' +
|
|
839
|
+
'supabase stripe playwright docker kubernetes k8s terraform rust go golang swift kotlin python typescript ' +
|
|
840
|
+
'javascript node express hono nest prisma drizzle trpc vitest jest graphql redis mongodb sqlite mysql aws gcp ' +
|
|
841
|
+
'azure vercel cloudflare lambda serverless webpack vite esbuild eslint prettier storybook electron tauri ' +
|
|
842
|
+
'flutter dart android ios expo redux zustand tanstack shadcn radix chakra mui css sass html seo a11y ' +
|
|
843
|
+
'accessibility auth oauth jwt websocket grpc openapi swagger cli git github ci cd pytest pandas numpy ' +
|
|
844
|
+
'pytorch tensorflow llm openai anthropic claude agent mcp skill skills frontend backend fullstack design ' +
|
|
845
|
+
'ui ux landing page component hooks form forms testing tests migration database schema api'
|
|
846
|
+
).split(' '),
|
|
847
|
+
);
|
|
848
|
+
|
|
849
|
+
/** Minimal keyword extractor (kept as an exported fallback; lib/score.js extractKeywords is the default). */
|
|
850
|
+
export function fallbackKeywords(prompt, { max = 4 } = {}) {
|
|
851
|
+
const counts = new Map();
|
|
852
|
+
for (const raw of String(prompt ?? '').toLowerCase().split(/[^a-z0-9+#.]+/)) {
|
|
853
|
+
const t = raw.replace(/^[.#+]+|[.#+]+$/g, '');
|
|
854
|
+
if (t.length < 2 || STOPWORDS.has(t) || /^\d+$/.test(t)) continue;
|
|
855
|
+
counts.set(t, (counts.get(t) ?? 0) + 1);
|
|
856
|
+
}
|
|
857
|
+
return [...counts.entries()]
|
|
858
|
+
.sort((a, b) => Number(TECH_LEXICON.has(b[0])) - Number(TECH_LEXICON.has(a[0])) || b[1] - a[1] || a[0].localeCompare(b[0]))
|
|
859
|
+
.slice(0, max)
|
|
860
|
+
.map((e) => e[0]);
|
|
861
|
+
}
|
|
862
|
+
|
|
863
|
+
async function getKeywords(prompt, d) {
|
|
864
|
+
let raw;
|
|
865
|
+
if (d.keywords) raw = await d.keywords(prompt);
|
|
866
|
+
else raw = extractKeywords(prompt, { max: 4 });
|
|
867
|
+
const out = [];
|
|
868
|
+
for (const k of Array.isArray(raw) ? raw : []) {
|
|
869
|
+
const t = String(k ?? '').trim().toLowerCase();
|
|
870
|
+
if (t.length >= 2 && !out.includes(t)) out.push(t);
|
|
871
|
+
if (out.length >= 4) break;
|
|
872
|
+
}
|
|
873
|
+
return out;
|
|
874
|
+
}
|
|
875
|
+
|
|
876
|
+
function repoWords(source) {
|
|
877
|
+
const repo = String(source).split('/')[1] ?? '';
|
|
878
|
+
return repo.replace(/[^A-Za-z0-9]+/g, ' ');
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
/**
|
|
882
|
+
* Rank gated candidates by relevance to the prompt, not installs alone.
|
|
883
|
+
* rankCandidates(cands, keywords, {prompt?, stackTags?, hits?} | Map)
|
|
884
|
+
* Each candidate becomes a pseudo-skill {name: skill, description: repo words} and is scored with the
|
|
885
|
+
* same BM25-lite as installed skills (lib/score.js) against the prompt plus its intent terms
|
|
886
|
+
* (expandIntent: "landing page" -> "frontend design"). A candidate is *matched* only when at least
|
|
887
|
+
* one prompt token matches exactly (stemmed) and the matched tokens are not all hyper-common words.
|
|
888
|
+
* rank = relevance + 0.5 * stack-tag matches (weak signal) + log10(installs + 1); unmatched
|
|
889
|
+
* candidates sink to the bottom with matches: 0. Returns new objects carrying
|
|
890
|
+
* {matches, matched, stackMatches, relevance, rank, hits}.
|
|
891
|
+
*/
|
|
892
|
+
export function rankCandidates(cands, keywords = [], opts = {}) {
|
|
893
|
+
const o = opts instanceof Map ? { hits: opts } : opts && typeof opts === 'object' ? opts : {};
|
|
894
|
+
const hitsByKey = o.hits instanceof Map ? o.hits : new Map();
|
|
895
|
+
const kws = (Array.isArray(keywords) ? keywords : []).map(lc).filter((k) => k.length >= 2);
|
|
896
|
+
const stackStems = new Set((Array.isArray(o.stackTags) ? o.stackTags : []).flatMap((t) => tokenize(String(t))));
|
|
897
|
+
const list = (Array.isArray(cands) ? cands : []).filter(isCandidate);
|
|
898
|
+
if (list.length === 0) return [];
|
|
899
|
+
|
|
900
|
+
const prompt = typeof o.prompt === 'string' && o.prompt.trim() ? o.prompt : kws.join(' ');
|
|
901
|
+
const intent = expandIntent(prompt);
|
|
902
|
+
const query = intent.length ? `${prompt} ${intent.join(' ')}` : prompt;
|
|
903
|
+
const pseudo = list.map((c, i) => ({ name: c.skill, description: repoWords(c.source), _i: i }));
|
|
904
|
+
const scored = new Map();
|
|
905
|
+
if (query.trim()) {
|
|
906
|
+
for (const s of scoreSkills(query, pseudo, { max: pseudo.length, minScore: MIN_CATALOG_MATCH_SCORE })) scored.set(s.skill._i, s);
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
const ranked = list.map((c, i) => {
|
|
910
|
+
const s = scored.get(i);
|
|
911
|
+
const matchedTokens = s ? s.matched : [];
|
|
912
|
+
const weak = isWeakMatch(matchedTokens);
|
|
913
|
+
const matches = weak ? 0 : matchedTokens.length;
|
|
914
|
+
const relevance = weak ? 0 : s.score;
|
|
915
|
+
const nameStems = new Set(tokenize(`${c.skill} ${repoWords(c.source)}`));
|
|
916
|
+
let stackMatches = 0;
|
|
917
|
+
for (const st of stackStems) if (nameStems.has(st)) stackMatches++;
|
|
918
|
+
const installsTerm = Math.log10(Math.max(0, Number(c.installs) || 0) + 1);
|
|
919
|
+
const rank = matches > 0 ? relevance + 0.5 * stackMatches + installsTerm : (0.5 * stackMatches + installsTerm) / 100;
|
|
920
|
+
const hits = Number(hitsByKey.get(lc(candidateRef(c)))) || 0;
|
|
921
|
+
return { ...c, matches, matched: weak ? [] : matchedTokens, stackMatches, relevance: Math.round(relevance * 1000) / 1000, rank: Math.round(rank * 1000) / 1000, hits };
|
|
922
|
+
});
|
|
923
|
+
ranked.sort((a, b) => Number(b.matches > 0) - Number(a.matches > 0) || b.rank - a.rank || b.hits - a.hits || byInstallsDesc(a, b));
|
|
924
|
+
return ranked;
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
export function searchCacheKey(q, owner = null) {
|
|
928
|
+
return owner ? `search:${q}|owner=${owner}` : `search:${q}`;
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
/** Negative-cache key for a failed request: `fail:catalog:<owner>` / `fail:search:<word>`. */
|
|
932
|
+
export function failKey(item) {
|
|
933
|
+
return `fail:${typeof item === 'string' ? item : item.key}`;
|
|
934
|
+
}
|
|
935
|
+
|
|
936
|
+
export function auditCacheKey(source, skill) {
|
|
937
|
+
return `audit:${source}@${skill}`;
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
/**
|
|
941
|
+
* The only prompt-derived words that may leave the machine from the per-prompt hook: single tokens that
|
|
942
|
+
* are in the built-in tech lexicon (react, stripe, supabase, ...), at most `max`, in keyword order.
|
|
943
|
+
* Free-form words (names, projects, verbs) are never shared.
|
|
944
|
+
*/
|
|
945
|
+
export function shareableKeywords(keywords, max = MAX_KEYWORD_SEARCHES) {
|
|
946
|
+
const out = [];
|
|
947
|
+
for (const k of Array.isArray(keywords) ? keywords : []) {
|
|
948
|
+
const t = lc(k).trim();
|
|
949
|
+
if (t.length >= 2 && SHAREABLE_LEXICON.has(t) && !out.includes(t)) out.push(t);
|
|
950
|
+
if (out.length >= max) break;
|
|
951
|
+
}
|
|
952
|
+
return out;
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
/**
|
|
956
|
+
* Query plan for an explicit search (`/singular find`): the joined keywords (+ stack tags to fill 4 terms)
|
|
957
|
+
* once unscoped, then scoped to each allowlisted owner. Kept small: at most 1 + owners requests.
|
|
958
|
+
* The per-prompt hook does not use this (it reads the cached owner catalogs instead).
|
|
959
|
+
*/
|
|
960
|
+
export function buildQueries(keywords, stackTags = [], allowlist = []) {
|
|
961
|
+
const kw = (Array.isArray(keywords) ? keywords : []).map(lc).filter((k) => k.length >= 2).slice(0, 4);
|
|
962
|
+
if (kw.length === 0) return [];
|
|
963
|
+
const joinedTerms = [...kw];
|
|
964
|
+
for (const t of Array.isArray(stackTags) ? stackTags : []) {
|
|
965
|
+
if (joinedTerms.length >= 4) break;
|
|
966
|
+
const tag = lc(t).trim();
|
|
967
|
+
if (tag.length >= 2 && !joinedTerms.includes(tag)) joinedTerms.push(tag);
|
|
968
|
+
}
|
|
969
|
+
const joined = joinedTerms.join(' ');
|
|
970
|
+
const out = [];
|
|
971
|
+
const seen = new Set();
|
|
972
|
+
const add = (q, owner = null) => {
|
|
973
|
+
const query = String(q).trim();
|
|
974
|
+
if (query.length < 2) return;
|
|
975
|
+
const key = `${lc(query)}|${owner ? lc(owner) : ''}`;
|
|
976
|
+
if (seen.has(key)) return;
|
|
977
|
+
seen.add(key);
|
|
978
|
+
out.push({ q: query, owner });
|
|
979
|
+
};
|
|
980
|
+
add(joined);
|
|
981
|
+
for (const owner of allowlistOwners(allowlist, 4)) add(joined, owner);
|
|
982
|
+
return out.slice(0, MAX_DISCOVER_QUERIES);
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
/**
|
|
986
|
+
* discover(prompt, {config, stackTags, timeoutMs, deps, installedNames, search})
|
|
987
|
+
* -> {candidates: Gated[], keywords, cached, rejected, queries, auditSkipped, errors, cooldownUntil, elapsedMs}
|
|
988
|
+
* Gated = Candidate & {owner, ref: "owner/repo@skill", matches, matched, rank, audit: {status, raw, providers}}
|
|
989
|
+
* Pipeline: keywords (local) -> owner catalogs (one request per allowlisted owner, cached cacheTtlHours,
|
|
990
|
+
* at most 2 in flight) -> up to MAX_KEYWORD_SEARCHES unscoped searches for tech-lexicon words
|
|
991
|
+
* only (`q=stripe`, cached per word) -> [search=true: one explicit search with the joined keywords]
|
|
992
|
+
* -> gate -> rank (prompt vs catalog names, stack tags weak, log10 installs) -> audit -> top maxCandidates.
|
|
993
|
+
* The only prompt-derived data on the wire is shareableKeywords() (tech nouns), plus the joined query
|
|
994
|
+
* when `search` is true (explicit /singular find).
|
|
995
|
+
* Respects the deadline: requests are skipped when < MIN_SEARCH_MS remain and audits when
|
|
996
|
+
* < MIN_AUDIT_MS remain (audit status then "unknown"). An HTTP 429 sets a 5-minute cooldown
|
|
997
|
+
* (cache key COOLDOWN_KEY) during which no request is made; cached catalogs still rank.
|
|
998
|
+
* `cached` is true when every catalog/search was served from the cache. Never throws.
|
|
999
|
+
*/
|
|
1000
|
+
export async function discover(prompt, opts = {}) {
|
|
1001
|
+
const { config = {}, stackTags = [], timeoutMs, deps = {}, installedNames = [], search = false } = opts;
|
|
1002
|
+
const d = withDeps(deps);
|
|
1003
|
+
const cfg = {
|
|
1004
|
+
allowlist: [],
|
|
1005
|
+
installFloor: 1000,
|
|
1006
|
+
maxCandidates: 2,
|
|
1007
|
+
remoteTimeoutMs: 2500,
|
|
1008
|
+
cacheTtlHours: 24,
|
|
1009
|
+
requireAuditPass: true,
|
|
1010
|
+
blockedRefs: [],
|
|
1011
|
+
...(config && typeof config === 'object' ? config : {}),
|
|
1012
|
+
};
|
|
1013
|
+
const budget = Number.isFinite(Number(timeoutMs)) ? Number(timeoutMs) : Math.max(Number(cfg.remoteTimeoutMs) * 2 || 0, 3000);
|
|
1014
|
+
const start = d.now();
|
|
1015
|
+
const deadline = start + budget;
|
|
1016
|
+
const remaining = () => deadline - d.now();
|
|
1017
|
+
const result = {
|
|
1018
|
+
candidates: [],
|
|
1019
|
+
keywords: [],
|
|
1020
|
+
intent: [],
|
|
1021
|
+
shared: [], // the prompt-derived words that were (or would be) sent: tech-lexicon nouns only
|
|
1022
|
+
cached: false,
|
|
1023
|
+
rejected: [],
|
|
1024
|
+
queries: [],
|
|
1025
|
+
auditSkipped: false,
|
|
1026
|
+
errors: [],
|
|
1027
|
+
cooldownUntil: null,
|
|
1028
|
+
elapsedMs: 0,
|
|
1029
|
+
};
|
|
1030
|
+
const finish = () => {
|
|
1031
|
+
result.elapsedMs = Math.max(0, d.now() - start);
|
|
1032
|
+
return result;
|
|
1033
|
+
};
|
|
1034
|
+
const safe = async (fn, label) => {
|
|
1035
|
+
try {
|
|
1036
|
+
return await fn();
|
|
1037
|
+
} catch (err) {
|
|
1038
|
+
result.errors.push(`${label}: ${String(err?.message ?? err)}`);
|
|
1039
|
+
return null;
|
|
1040
|
+
}
|
|
1041
|
+
};
|
|
1042
|
+
|
|
1043
|
+
try {
|
|
1044
|
+
result.keywords = await getKeywords(prompt, d);
|
|
1045
|
+
result.intent = expandIntent(prompt);
|
|
1046
|
+
if (result.keywords.length === 0 && result.intent.length === 0) return finish();
|
|
1047
|
+
const owners = allowlistOwners(cfg.allowlist);
|
|
1048
|
+
if (owners.length === 0 && !search) return finish();
|
|
1049
|
+
const cache = getCache(d);
|
|
1050
|
+
const ttl = Number(cfg.cacheTtlHours) || 0;
|
|
1051
|
+
const perRequest = (left) => Math.max(1, Math.min(Number(cfg.remoteTimeoutMs) || 2500, left - 50));
|
|
1052
|
+
let cooldownUntil = await getCooldown(cache, d.now());
|
|
1053
|
+
result.cooldownUntil = cooldownUntil || null;
|
|
1054
|
+
const rateLimited = async () => {
|
|
1055
|
+
if (!cooldownUntil) {
|
|
1056
|
+
cooldownUntil = await setCooldown(cache, d.now());
|
|
1057
|
+
result.cooldownUntil = cooldownUntil;
|
|
1058
|
+
result.errors.push(`skills.sh rate limited (429); no requests until ${new Date(cooldownUntil).toISOString()}`);
|
|
1059
|
+
}
|
|
1060
|
+
};
|
|
1061
|
+
|
|
1062
|
+
// --- request plan: one catalog per owner (cached), tech-noun searches (cached per word), plus one
|
|
1063
|
+
// explicit joined search when asked (/singular find). Catalogs come first so a tight deadline drops
|
|
1064
|
+
// the keyword searches, not the (usually cached) catalogs.
|
|
1065
|
+
const plan = owners.map((owner) => ({ kind: 'catalog', q: CATALOG_QUERY, owner, key: catalogCacheKey(owner) }));
|
|
1066
|
+
const joined = search ? buildQueries(result.keywords, stackTags, [])[0]?.q ?? null : null;
|
|
1067
|
+
result.shared = shareableKeywords(result.keywords).filter((kw) => kw !== joined);
|
|
1068
|
+
for (const kw of result.shared) plan.push({ kind: 'keyword', q: kw, owner: null, key: searchCacheKey(kw) });
|
|
1069
|
+
if (joined) plan.push({ kind: 'search', q: joined, owner: null, key: searchCacheKey(joined) });
|
|
1070
|
+
|
|
1071
|
+
const fetched = await mapLimit(plan, MAX_CONCURRENT_REQUESTS, async (item) => {
|
|
1072
|
+
const hit = await safe(() => cache.get(item.key, ttl), `cacheGet ${item.key}`);
|
|
1073
|
+
if (Array.isArray(hit)) return { ...item, candidates: hit, cached: true, stale: false, skipped: null };
|
|
1074
|
+
// Expired copy (stale-while-revalidate) and the failure marker for this exact request.
|
|
1075
|
+
const staleHit = await safe(() => cache.get(item.key, STALE_TTL_HOURS), `cacheGet ${item.key}`);
|
|
1076
|
+
const stale = Array.isArray(staleHit) ? staleHit : null;
|
|
1077
|
+
const withStale = (skipped) => ({ ...item, candidates: stale ?? [], cached: stale !== null, stale: stale !== null, skipped });
|
|
1078
|
+
const marker = await safe(() => cache.get(failKey(item), FAIL_MARKER_MS / 3600000), `cacheGet ${failKey(item)}`);
|
|
1079
|
+
if (marker && typeof marker === 'object' && Number(marker.until) > d.now()) return withStale('recent-failure');
|
|
1080
|
+
if (cooldownUntil) return withStale('cooldown');
|
|
1081
|
+
const left = remaining();
|
|
1082
|
+
if (left < MIN_SEARCH_MS) return withStale('deadline');
|
|
1083
|
+
const r = await searchSkillsRaw(item.q, { limit: MAX_SEARCH_LIMIT, timeoutMs: perRequest(left), owner: item.owner }, d);
|
|
1084
|
+
if (r.ok) {
|
|
1085
|
+
await safe(() => cache.set(item.key, r.candidates), `cacheSet ${item.key}`);
|
|
1086
|
+
return { ...item, candidates: r.candidates, cached: false, stale: false, skipped: null, truncated: r.truncated === true };
|
|
1087
|
+
}
|
|
1088
|
+
if (r.status === 429) await rateLimited();
|
|
1089
|
+
else await safe(() => cache.set(failKey(item), { until: d.now() + FAIL_MARKER_MS, error: String(r.error ?? '').slice(0, 120), ts: d.now() }), `cacheSet ${failKey(item)}`);
|
|
1090
|
+
result.errors.push(`${item.kind} "${item.q}"${item.owner ? ` owner=${item.owner}` : ''}: ${r.error}${stale ? ' (stale cached copy used)' : ''}`);
|
|
1091
|
+
return withStale(r.status === 429 ? 'rate-limited' : 'error');
|
|
1092
|
+
});
|
|
1093
|
+
result.queries = fetched.map(({ kind, q, owner, cached, stale, skipped, candidates, truncated }) => ({ kind, q, owner, cached, stale: stale === true, skipped, results: candidates.length, truncated: truncated === true }));
|
|
1094
|
+
result.cached = fetched.length > 0 && fetched.every((s) => s.cached);
|
|
1095
|
+
|
|
1096
|
+
// --- gate, then rank by relevance (prompt vs catalog names, stack tags weak, installs)
|
|
1097
|
+
const hits = new Map();
|
|
1098
|
+
for (const s of fetched) {
|
|
1099
|
+
const seenHere = new Set();
|
|
1100
|
+
for (const c of s.candidates) {
|
|
1101
|
+
if (!isCandidate(c)) continue;
|
|
1102
|
+
const key = lc(candidateRef(c));
|
|
1103
|
+
if (seenHere.has(key)) continue;
|
|
1104
|
+
seenHere.add(key);
|
|
1105
|
+
hits.set(key, (hits.get(key) ?? 0) + 1);
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
const merged = mergeCandidates(fetched.map((s) => s.candidates));
|
|
1109
|
+
const gate = gateCandidates(merged, cfg, { installedNames });
|
|
1110
|
+
result.rejected = gate.rejected;
|
|
1111
|
+
const maxCandidates = Math.max(0, Math.floor(Number(cfg.maxCandidates) || 0));
|
|
1112
|
+
if (maxCandidates === 0 || gate.accepted.length === 0) return finish();
|
|
1113
|
+
const ranked = rankCandidates(gate.accepted, result.keywords, { prompt, stackTags, hits }).filter((c) => c.matches > 0);
|
|
1114
|
+
if (ranked.length === 0) return finish();
|
|
1115
|
+
|
|
1116
|
+
// --- audit top N (+2 spare so a fail can be backfilled without a second round trip)
|
|
1117
|
+
const top = ranked.slice(0, maxCandidates + 2);
|
|
1118
|
+
const gated = await mapLimit(top, MAX_CONCURRENT_REQUESTS, async (cand) => {
|
|
1119
|
+
const ref = candidateRef(cand);
|
|
1120
|
+
const key = auditCacheKey(cand.source, cand.skill);
|
|
1121
|
+
let audit = await safe(() => cache.get(key, ttl), `cacheGet ${key}`);
|
|
1122
|
+
if (!audit || typeof audit !== 'object' || typeof audit.status !== 'string') {
|
|
1123
|
+
const left = remaining();
|
|
1124
|
+
if (cooldownUntil) {
|
|
1125
|
+
audit = { status: 'unknown', raw: { skipped: 'cooldown' }, providers: [] };
|
|
1126
|
+
} else if (left < MIN_AUDIT_MS) {
|
|
1127
|
+
result.auditSkipped = true;
|
|
1128
|
+
audit = { status: 'unknown', raw: { skipped: 'deadline' }, providers: [] };
|
|
1129
|
+
} else {
|
|
1130
|
+
audit = await auditSkill(cand.source, cand.skill, { timeoutMs: perRequest(left) }, d);
|
|
1131
|
+
if (audit.raw?.rateLimited) await rateLimited();
|
|
1132
|
+
if (!audit.raw?.transient) await safe(() => cache.set(key, audit), `cacheSet ${key}`);
|
|
1133
|
+
}
|
|
1134
|
+
}
|
|
1135
|
+
return { ...cand, owner: cand.source.split('/')[0], ref, audit };
|
|
1136
|
+
});
|
|
1137
|
+
|
|
1138
|
+
const kept = [];
|
|
1139
|
+
for (const g of gated) {
|
|
1140
|
+
if (cfg.requireAuditPass && g.audit.status === 'fail') {
|
|
1141
|
+
result.rejected.unshift({ cand: g, reason: `audit fail (${g.audit.providers.filter((p) => p.status === 'fail' || p.riskLevel === 'CRITICAL').map((p) => p.provider).join(', ') || 'provider'})` });
|
|
1142
|
+
continue;
|
|
1143
|
+
}
|
|
1144
|
+
kept.push(g);
|
|
1145
|
+
}
|
|
1146
|
+
result.candidates = kept.slice(0, maxCandidates);
|
|
1147
|
+
return finish();
|
|
1148
|
+
} catch (err) {
|
|
1149
|
+
result.errors.push(`discover: ${String(err?.message ?? err)}`);
|
|
1150
|
+
return finish();
|
|
1151
|
+
}
|
|
1152
|
+
}
|