skilld 1.5.5 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -23
- package/dist/_chunks/agent.mjs +2 -78
- package/dist/_chunks/agent.mjs.map +1 -1
- package/dist/_chunks/assemble.mjs +1 -18
- package/dist/_chunks/assemble.mjs.map +1 -1
- package/dist/_chunks/author-group.mjs +17 -0
- package/dist/_chunks/author-group.mjs.map +1 -0
- package/dist/_chunks/author.mjs +8 -24
- package/dist/_chunks/author.mjs.map +1 -1
- package/dist/_chunks/cache.mjs +1 -73
- package/dist/_chunks/cache.mjs.map +1 -1
- package/dist/_chunks/cache2.mjs +84 -17
- package/dist/_chunks/cache2.mjs.map +1 -1
- package/dist/_chunks/cli-helpers.mjs +3 -166
- package/dist/_chunks/cli-helpers.mjs.map +1 -1
- package/dist/_chunks/cli-helpers2.mjs +0 -11
- package/dist/_chunks/config.mjs +119 -54
- package/dist/_chunks/config.mjs.map +1 -1
- package/dist/_chunks/core.mjs +9 -0
- package/dist/_chunks/detect.mjs +29 -226
- package/dist/_chunks/detect.mjs.map +1 -1
- package/dist/_chunks/embedding-cache.mjs +0 -5
- package/dist/_chunks/embedding-cache2.mjs +2 -3
- package/dist/_chunks/formatting.mjs +0 -6
- package/dist/_chunks/formatting.mjs.map +1 -1
- package/dist/_chunks/index.d.mts +0 -10
- package/dist/_chunks/index.d.mts.map +1 -1
- package/dist/_chunks/index2.d.mts +3 -6
- package/dist/_chunks/index2.d.mts.map +1 -1
- package/dist/_chunks/index3.d.mts +81 -109
- package/dist/_chunks/index3.d.mts.map +1 -1
- package/dist/_chunks/install.mjs +85 -550
- package/dist/_chunks/install.mjs.map +1 -1
- package/dist/_chunks/install2.mjs +554 -0
- package/dist/_chunks/install2.mjs.map +1 -0
- package/dist/_chunks/libs/@sinclair/typebox.mjs +0 -444
- package/dist/_chunks/libs/@sinclair/typebox.mjs.map +1 -1
- package/dist/_chunks/list.mjs +0 -16
- package/dist/_chunks/list.mjs.map +1 -1
- package/dist/_chunks/lockfile.mjs +2 -10
- package/dist/_chunks/lockfile.mjs.map +1 -1
- package/dist/_chunks/markdown.mjs +0 -9
- package/dist/_chunks/markdown.mjs.map +1 -1
- package/dist/_chunks/package-json.mjs +0 -25
- package/dist/_chunks/package-json.mjs.map +1 -1
- package/dist/_chunks/package-registry.mjs +465 -0
- package/dist/_chunks/package-registry.mjs.map +1 -0
- package/dist/_chunks/pool2.mjs +0 -2
- package/dist/_chunks/pool2.mjs.map +1 -1
- package/dist/_chunks/prefix.mjs +108 -0
- package/dist/_chunks/prefix.mjs.map +1 -0
- package/dist/_chunks/prepare.mjs +14 -9
- package/dist/_chunks/prepare.mjs.map +1 -1
- package/dist/_chunks/prepare2.mjs +1 -19
- package/dist/_chunks/prepare2.mjs.map +1 -1
- package/dist/_chunks/prompts.mjs +6 -201
- package/dist/_chunks/prompts.mjs.map +1 -1
- package/dist/_chunks/retriv.mjs +23 -24
- package/dist/_chunks/retriv.mjs.map +1 -1
- package/dist/_chunks/rolldown-runtime.mjs +0 -2
- package/dist/_chunks/sanitize.mjs +0 -78
- package/dist/_chunks/sanitize.mjs.map +1 -1
- package/dist/_chunks/search-helpers.mjs +99 -0
- package/dist/_chunks/search-helpers.mjs.map +1 -0
- package/dist/_chunks/search-interactive.mjs +1 -18
- package/dist/_chunks/search-interactive.mjs.map +1 -1
- package/dist/_chunks/search.mjs +218 -19
- package/dist/_chunks/search.mjs.map +1 -0
- package/dist/_chunks/setup.mjs +0 -13
- package/dist/_chunks/setup.mjs.map +1 -1
- package/dist/_chunks/shared.mjs +1 -473
- package/dist/_chunks/shared.mjs.map +1 -1
- package/dist/_chunks/skills.mjs +3 -3
- package/dist/_chunks/skills.mjs.map +1 -1
- package/dist/_chunks/sources.mjs +1179 -1440
- package/dist/_chunks/sources.mjs.map +1 -1
- package/dist/_chunks/sync-registry.mjs +59 -0
- package/dist/_chunks/sync-registry.mjs.map +1 -0
- package/dist/_chunks/sync-shared.mjs +0 -16
- package/dist/_chunks/sync-shared2.mjs +10 -49
- package/dist/_chunks/sync-shared2.mjs.map +1 -1
- package/dist/_chunks/sync.mjs +209 -120
- package/dist/_chunks/sync.mjs.map +1 -1
- package/dist/_chunks/sync2.mjs +1 -21
- package/dist/_chunks/types.d.mts +0 -2
- package/dist/_chunks/types.d.mts.map +1 -1
- package/dist/_chunks/uninstall.mjs +3 -27
- package/dist/_chunks/uninstall.mjs.map +1 -1
- package/dist/_chunks/upload.mjs +152 -0
- package/dist/_chunks/upload.mjs.map +1 -0
- package/dist/_chunks/validate.mjs +1 -8
- package/dist/_chunks/validate.mjs.map +1 -1
- package/dist/_chunks/version.mjs +30 -0
- package/dist/_chunks/version.mjs.map +1 -0
- package/dist/_chunks/wizard.mjs +2 -3
- package/dist/_chunks/yaml.mjs +0 -21
- package/dist/_chunks/yaml.mjs.map +1 -1
- package/dist/agent/index.d.mts +0 -24
- package/dist/agent/index.d.mts.map +1 -1
- package/dist/agent/index.mjs +2 -9
- package/dist/cache/index.mjs +1 -3
- package/dist/cli-entry.mjs +0 -6
- package/dist/cli-entry.mjs.map +1 -1
- package/dist/cli.mjs +48 -33
- package/dist/cli.mjs.map +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/index.mjs +2 -8
- package/dist/prepare.mjs +0 -12
- package/dist/prepare.mjs.map +1 -1
- package/dist/retriv/index.mjs +0 -2
- package/dist/retriv/worker.d.mts +0 -3
- package/dist/retriv/worker.d.mts.map +1 -1
- package/dist/retriv/worker.mjs +0 -2
- package/dist/retriv/worker.mjs.map +1 -1
- package/dist/sources/index.d.mts +2 -2
- package/dist/sources/index.mjs +3 -7
- package/dist/types.d.mts +1 -1
- package/package.json +20 -21
- package/dist/_chunks/search2.mjs +0 -319
- package/dist/_chunks/search2.mjs.map +0 -1
package/dist/_chunks/sources.mjs
CHANGED
|
@@ -1,25 +1,22 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { t as getCacheDir } from "./version.mjs";
|
|
2
2
|
import { i as readPackageJsonSafe } from "./package-json.mjs";
|
|
3
3
|
import { t as yamlEscape } from "./yaml.mjs";
|
|
4
4
|
import { i as parseFrontmatter, n as extractLinks, r as extractTitle, t as extractDescription } from "./markdown.mjs";
|
|
5
|
-
import {
|
|
5
|
+
import { n as getCrawlUrl, r as getDocOverride, t as getBlogPreset } from "./package-registry.mjs";
|
|
6
|
+
import { r as mapInsert } from "./shared.mjs";
|
|
6
7
|
import { tmpdir } from "node:os";
|
|
7
8
|
import { basename, dirname, join, resolve } from "pathe";
|
|
8
9
|
import { createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, rmSync } from "node:fs";
|
|
9
10
|
import { htmlToMarkdown } from "mdream";
|
|
11
|
+
import pLimit from "p-limit";
|
|
10
12
|
import { spawnSync } from "node:child_process";
|
|
11
13
|
import { ofetch } from "ofetch";
|
|
14
|
+
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
12
15
|
import { crawlAndGenerate } from "@mdream/crawl";
|
|
13
16
|
import { glob } from "tinyglobby";
|
|
14
17
|
import { downloadTemplate } from "giget";
|
|
15
|
-
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
16
|
-
import pLimit from "p-limit";
|
|
17
18
|
import { Writable } from "node:stream";
|
|
18
19
|
import { resolvePathSync } from "mlly";
|
|
19
|
-
//#region src/sources/github-common.ts
|
|
20
|
-
/**
|
|
21
|
-
* Shared constants and helpers for GitHub source modules (issues, discussions, releases)
|
|
22
|
-
*/
|
|
23
20
|
const BOT_USERS = new Set([
|
|
24
21
|
"renovate[bot]",
|
|
25
22
|
"dependabot[bot]",
|
|
@@ -27,25 +24,17 @@ const BOT_USERS = new Set([
|
|
|
27
24
|
"dependabot",
|
|
28
25
|
"github-actions[bot]"
|
|
29
26
|
]);
|
|
30
|
-
/** Extract YYYY-MM-DD date from an ISO timestamp */
|
|
31
27
|
const isoDate = (iso) => iso.split("T")[0];
|
|
32
|
-
/** Build YAML frontmatter from a key-value object, auto-quoting strings with special chars */
|
|
33
28
|
function buildFrontmatter(fields) {
|
|
34
29
|
const lines = ["---"];
|
|
35
30
|
for (const [k, v] of Object.entries(fields)) if (v !== void 0) lines.push(`${k}: ${typeof v === "string" ? yamlEscape(v) : v}`);
|
|
36
31
|
lines.push("---");
|
|
37
32
|
return lines.join("\n");
|
|
38
33
|
}
|
|
39
|
-
/** Check if body contains a code block */
|
|
40
34
|
function hasCodeBlock(text) {
|
|
41
35
|
return /```[\s\S]*?```/.test(text) || /`[^`]+`/.test(text);
|
|
42
36
|
}
|
|
43
|
-
/** Noise patterns in comments — filter these out */
|
|
44
37
|
const COMMENT_NOISE_RE = /^(?:\+1|👍|same here|any update|bump|following|is there any progress|when will this|me too|i have the same|same issue|thanks|thank you)[\s!?.]*$/i;
|
|
45
|
-
/**
|
|
46
|
-
* Smart body truncation — preserves code blocks and error messages.
|
|
47
|
-
* Instead of slicing at a char limit, finds a safe break point.
|
|
48
|
-
*/
|
|
49
38
|
function truncateBody(body, limit) {
|
|
50
39
|
if (body.length <= limit) return body;
|
|
51
40
|
const codeBlockRe = /```[\s\S]*?```/g;
|
|
@@ -66,10 +55,6 @@ function truncateBody(body, limit) {
|
|
|
66
55
|
return `${slice}...`;
|
|
67
56
|
}
|
|
68
57
|
let _ghToken;
|
|
69
|
-
/**
|
|
70
|
-
* Get GitHub auth token from gh CLI (cached).
|
|
71
|
-
* Returns null if gh CLI is not available or not authenticated.
|
|
72
|
-
*/
|
|
73
58
|
function getGitHubToken() {
|
|
74
59
|
if (_ghToken !== void 0) return _ghToken;
|
|
75
60
|
try {
|
|
@@ -88,13 +73,10 @@ function getGitHubToken() {
|
|
|
88
73
|
}
|
|
89
74
|
return _ghToken;
|
|
90
75
|
}
|
|
91
|
-
/** Repos where ungh.cc failed but gh api succeeded (likely private) */
|
|
92
76
|
const _needsAuth = /* @__PURE__ */ new Set();
|
|
93
|
-
/** Mark a repo as needing authenticated access */
|
|
94
77
|
function markRepoPrivate(owner, repo) {
|
|
95
78
|
_needsAuth.add(`${owner}/${repo}`);
|
|
96
79
|
}
|
|
97
|
-
/** Check if a repo is known to need authenticated access */
|
|
98
80
|
function isKnownPrivateRepo(owner, repo) {
|
|
99
81
|
return _needsAuth.has(`${owner}/${repo}`);
|
|
100
82
|
}
|
|
@@ -106,24 +88,15 @@ const ghApiFetch = ofetch.create({
|
|
|
106
88
|
headers: { "User-Agent": "skilld/1.0" }
|
|
107
89
|
});
|
|
108
90
|
const LINK_NEXT_RE = /<([^>]+)>;\s*rel="next"/;
|
|
109
|
-
/** Parse GitHub Link header for next page URL */
|
|
110
91
|
function parseLinkNext(header) {
|
|
111
92
|
if (!header) return null;
|
|
112
93
|
return header.match(LINK_NEXT_RE)?.[1] ?? null;
|
|
113
94
|
}
|
|
114
|
-
/**
|
|
115
|
-
* Authenticated fetch against api.github.com. Returns null if no token or request fails.
|
|
116
|
-
* Endpoint should be relative, e.g. `repos/owner/repo/releases`.
|
|
117
|
-
*/
|
|
118
95
|
async function ghApi(endpoint) {
|
|
119
96
|
const token = getGitHubToken();
|
|
120
97
|
if (!token) return null;
|
|
121
98
|
return ghApiFetch(`${GH_API}/${endpoint}`, { headers: { Authorization: `token ${token}` } }).catch(() => null);
|
|
122
99
|
}
|
|
123
|
-
/**
|
|
124
|
-
* Paginated GitHub API fetch. Follows Link headers, returns concatenated arrays.
|
|
125
|
-
* Endpoint should return a JSON array, e.g. `repos/owner/repo/releases`.
|
|
126
|
-
*/
|
|
127
100
|
async function ghApiPaginated(endpoint) {
|
|
128
101
|
const token = getGitHubToken();
|
|
129
102
|
if (!token) return [];
|
|
@@ -138,25 +111,40 @@ async function ghApiPaginated(endpoint) {
|
|
|
138
111
|
}
|
|
139
112
|
return results;
|
|
140
113
|
}
|
|
141
|
-
|
|
142
|
-
//#region src/sources/utils.ts
|
|
143
|
-
/**
|
|
144
|
-
* Shared utilities for doc resolution
|
|
145
|
-
*/
|
|
114
|
+
const SKILLD_USER_AGENT = "skilld/1.0 (+https://github.com/harlan-zw/skilld)";
|
|
146
115
|
const $fetch = ofetch.create({
|
|
147
116
|
retry: 3,
|
|
148
|
-
retryDelay:
|
|
117
|
+
retryDelay: 1e3,
|
|
118
|
+
retryStatusCodes: [
|
|
119
|
+
408,
|
|
120
|
+
429,
|
|
121
|
+
500,
|
|
122
|
+
502,
|
|
123
|
+
503,
|
|
124
|
+
504
|
|
125
|
+
],
|
|
149
126
|
timeout: 15e3,
|
|
150
|
-
headers: { "User-Agent":
|
|
127
|
+
headers: { "User-Agent": SKILLD_USER_AGENT }
|
|
151
128
|
});
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
129
|
+
function createRateLimitedRunner(intervalMs) {
|
|
130
|
+
let queue = Promise.resolve();
|
|
131
|
+
let lastRunAt = 0;
|
|
132
|
+
return async function runRateLimited(task) {
|
|
133
|
+
const run = async () => {
|
|
134
|
+
const waitMs = intervalMs - (Date.now() - lastRunAt);
|
|
135
|
+
if (waitMs > 0) await new Promise((resolve) => setTimeout(resolve, waitMs));
|
|
136
|
+
lastRunAt = Date.now();
|
|
137
|
+
return task();
|
|
138
|
+
};
|
|
139
|
+
const request = queue.then(run, run);
|
|
140
|
+
queue = request.then(() => void 0, () => void 0);
|
|
141
|
+
return request;
|
|
142
|
+
};
|
|
143
|
+
}
|
|
155
144
|
async function fetchText(url) {
|
|
156
145
|
return $fetch(url, { responseType: "text" }).catch(() => null);
|
|
157
146
|
}
|
|
158
147
|
const RAW_GH_RE = /raw\.githubusercontent\.com\/([^/]+)\/([^/]+)/;
|
|
159
|
-
/** Extract owner/repo from a GitHub raw content URL */
|
|
160
148
|
function extractGitHubRepo(url) {
|
|
161
149
|
const match = url.match(RAW_GH_RE);
|
|
162
150
|
return match ? {
|
|
@@ -164,14 +152,6 @@ function extractGitHubRepo(url) {
|
|
|
164
152
|
repo: match[2]
|
|
165
153
|
} : null;
|
|
166
154
|
}
|
|
167
|
-
/**
|
|
168
|
-
* Fetch text from a GitHub raw URL with auth fallback for private repos.
|
|
169
|
-
* Tries unauthenticated first (fast path), falls back to authenticated
|
|
170
|
-
* request when the repo is known to be private or unauthenticated fails.
|
|
171
|
-
*
|
|
172
|
-
* Only sends auth tokens to raw.githubusercontent.com — returns null for
|
|
173
|
-
* non-GitHub URLs that fail unauthenticated to prevent token leakage.
|
|
174
|
-
*/
|
|
175
155
|
async function fetchGitHubRaw(url) {
|
|
176
156
|
const gh = extractGitHubRepo(url);
|
|
177
157
|
if (!(gh ? isKnownPrivateRepo(gh.owner, gh.repo) : false)) {
|
|
@@ -188,17 +168,11 @@ async function fetchGitHubRaw(url) {
|
|
|
188
168
|
if (content) markRepoPrivate(gh.owner, gh.repo);
|
|
189
169
|
return content;
|
|
190
170
|
}
|
|
191
|
-
/**
|
|
192
|
-
* Verify URL exists and is not HTML (likely 404 page)
|
|
193
|
-
*/
|
|
194
171
|
async function verifyUrl(url) {
|
|
195
172
|
const res = await $fetch.raw(url, { method: "HEAD" }).catch(() => null);
|
|
196
173
|
if (!res) return false;
|
|
197
174
|
return !(res.headers.get("content-type") || "").includes("text/html");
|
|
198
175
|
}
|
|
199
|
-
/**
|
|
200
|
-
* Check if URL points to a social media or package registry site (not real docs)
|
|
201
|
-
*/
|
|
202
176
|
const USELESS_HOSTS = new Set([
|
|
203
177
|
"twitter.com",
|
|
204
178
|
"x.com",
|
|
@@ -218,9 +192,6 @@ function isUselessDocsUrl(url) {
|
|
|
218
192
|
return false;
|
|
219
193
|
}
|
|
220
194
|
}
|
|
221
|
-
/**
|
|
222
|
-
* Check if URL is a GitHub repo URL (not a docs site)
|
|
223
|
-
*/
|
|
224
195
|
function isGitHubRepoUrl(url) {
|
|
225
196
|
try {
|
|
226
197
|
const parsed = new URL(url);
|
|
@@ -229,9 +200,20 @@ function isGitHubRepoUrl(url) {
|
|
|
229
200
|
return false;
|
|
230
201
|
}
|
|
231
202
|
}
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
203
|
+
function isLikelyCodeHostUrl(url) {
|
|
204
|
+
if (!url) return false;
|
|
205
|
+
try {
|
|
206
|
+
const parsed = new URL(url);
|
|
207
|
+
return [
|
|
208
|
+
"github.com",
|
|
209
|
+
"www.github.com",
|
|
210
|
+
"gitlab.com",
|
|
211
|
+
"www.gitlab.com"
|
|
212
|
+
].includes(parsed.hostname);
|
|
213
|
+
} catch {
|
|
214
|
+
return false;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
235
217
|
function parseGitHubUrl(url) {
|
|
236
218
|
const match = url.match(/github\.com\/([^/]+)\/([^/]+?)(?:\.git)?(?:[/#]|$)/);
|
|
237
219
|
if (!match) return null;
|
|
@@ -240,16 +222,9 @@ function parseGitHubUrl(url) {
|
|
|
240
222
|
repo: match[2]
|
|
241
223
|
};
|
|
242
224
|
}
|
|
243
|
-
/**
|
|
244
|
-
* Normalize git repo URL to https
|
|
245
|
-
*/
|
|
246
225
|
function normalizeRepoUrl(url) {
|
|
247
226
|
return url.replace(/^git\+/, "").replace(/#.*$/, "").replace(/\.git$/, "").replace(/^git:\/\//, "https://").replace(/^ssh:\/\/git@github\.com/, "https://github.com").replace(/^git@github\.com:/, "https://github.com/");
|
|
248
227
|
}
|
|
249
|
-
/**
|
|
250
|
-
* Parse package spec with optional dist-tag or version: "vue@beta" → { name: "vue", tag: "beta" }
|
|
251
|
-
* Handles scoped packages: "@vue/reactivity@beta" → { name: "@vue/reactivity", tag: "beta" }
|
|
252
|
-
*/
|
|
253
228
|
function parsePackageSpec(spec) {
|
|
254
229
|
if (spec.startsWith("@")) {
|
|
255
230
|
const slashIdx = spec.indexOf("/");
|
|
@@ -269,9 +244,6 @@ function parsePackageSpec(spec) {
|
|
|
269
244
|
};
|
|
270
245
|
return { name: spec };
|
|
271
246
|
}
|
|
272
|
-
/**
|
|
273
|
-
* Extract branch hint from URL fragment (e.g. "git+https://...#main" → "main")
|
|
274
|
-
*/
|
|
275
247
|
function extractBranchHint(url) {
|
|
276
248
|
const hash = url.indexOf("#");
|
|
277
249
|
if (hash === -1) return void 0;
|
|
@@ -279,11 +251,6 @@ function extractBranchHint(url) {
|
|
|
279
251
|
if (!fragment || fragment === "readme") return void 0;
|
|
280
252
|
return fragment;
|
|
281
253
|
}
|
|
282
|
-
//#endregion
|
|
283
|
-
//#region src/sources/releases.ts
|
|
284
|
-
/**
|
|
285
|
-
* GitHub release notes fetching via GitHub API (preferred) with ungh.cc fallback
|
|
286
|
-
*/
|
|
287
254
|
function parseSemver(version) {
|
|
288
255
|
const clean = version.replace(/^v/, "");
|
|
289
256
|
const match = clean.match(/^(\d+)(?:\.(\d+))?(?:\.(\d+))?/);
|
|
@@ -295,13 +262,6 @@ function parseSemver(version) {
|
|
|
295
262
|
raw: clean
|
|
296
263
|
};
|
|
297
264
|
}
|
|
298
|
-
/**
|
|
299
|
-
* Extract version from a release tag, handling monorepo formats:
|
|
300
|
-
* - `pkg@1.2.3` → `1.2.3`
|
|
301
|
-
* - `pkg-v1.2.3` → `1.2.3`
|
|
302
|
-
* - `v1.2.3` → `1.2.3`
|
|
303
|
-
* - `1.2.3` → `1.2.3`
|
|
304
|
-
*/
|
|
305
265
|
function extractVersion(tag, packageName) {
|
|
306
266
|
if (packageName) {
|
|
307
267
|
const atMatch = tag.match(new RegExp(`^${escapeRegex(packageName)}@(.+)$`));
|
|
@@ -314,15 +274,9 @@ function extractVersion(tag, packageName) {
|
|
|
314
274
|
function escapeRegex(str) {
|
|
315
275
|
return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
316
276
|
}
|
|
317
|
-
/**
|
|
318
|
-
* Check if a release tag belongs to a specific package
|
|
319
|
-
*/
|
|
320
277
|
function tagMatchesPackage(tag, packageName) {
|
|
321
278
|
return tag.startsWith(`${packageName}@`) || tag.startsWith(`${packageName}-v`) || tag.startsWith(`${packageName}-`);
|
|
322
279
|
}
|
|
323
|
-
/**
|
|
324
|
-
* Check if a version string contains a prerelease suffix (e.g. 6.0.0-beta, 1.2.3-rc.1)
|
|
325
|
-
*/
|
|
326
280
|
function isPrerelease(version) {
|
|
327
281
|
return /^\d+\.\d+\.\d+-.+/.test(version.replace(/^v/, ""));
|
|
328
282
|
}
|
|
@@ -331,7 +285,6 @@ function compareSemver(a, b) {
|
|
|
331
285
|
if (a.minor !== b.minor) return a.minor - b.minor;
|
|
332
286
|
return a.patch - b.patch;
|
|
333
287
|
}
|
|
334
|
-
/** Map GitHub API release to our GitHubRelease shape */
|
|
335
288
|
function mapApiRelease(r) {
|
|
336
289
|
return {
|
|
337
290
|
id: r.id,
|
|
@@ -343,20 +296,11 @@ function mapApiRelease(r) {
|
|
|
343
296
|
markdown: r.body
|
|
344
297
|
};
|
|
345
298
|
}
|
|
346
|
-
/**
|
|
347
|
-
* Fetch all releases — GitHub API first (authenticated, async), ungh.cc fallback
|
|
348
|
-
*/
|
|
349
299
|
async function fetchAllReleases(owner, repo) {
|
|
350
300
|
const apiReleases = await ghApiPaginated(`repos/${owner}/${repo}/releases`);
|
|
351
301
|
if (apiReleases.length > 0) return apiReleases.map(mapApiRelease);
|
|
352
302
|
return (await $fetch(`https://ungh.cc/repos/${owner}/${repo}/releases`, { signal: AbortSignal.timeout(15e3) }).catch(() => null))?.releases ?? [];
|
|
353
303
|
}
|
|
354
|
-
/**
|
|
355
|
-
* Select last 20 stable releases for a package, sorted newest first.
|
|
356
|
-
* For monorepos, filters to package-specific tags (pkg@version).
|
|
357
|
-
* Falls back to generic tags (v1.2.3) only if no package-specific found.
|
|
358
|
-
* If installedVersion is provided, filters out releases newer than it.
|
|
359
|
-
*/
|
|
360
304
|
function selectReleases(releases, packageName, installedVersion, fromDate) {
|
|
361
305
|
const hasMonorepoTags = packageName && releases.some((r) => tagMatchesPackage(r.tag, packageName));
|
|
362
306
|
const installedSv = installedVersion ? parseSemver(installedVersion) : null;
|
|
@@ -386,9 +330,6 @@ function selectReleases(releases, packageName, installedVersion, fromDate) {
|
|
|
386
330
|
});
|
|
387
331
|
return fromDate ? sorted : sorted.slice(0, 20);
|
|
388
332
|
}
|
|
389
|
-
/**
|
|
390
|
-
* Format a release as markdown with YAML frontmatter
|
|
391
|
-
*/
|
|
392
333
|
function formatRelease(release, packageName) {
|
|
393
334
|
const date = isoDate(release.publishedAt || release.createdAt);
|
|
394
335
|
const version = extractVersion(release.tag, packageName) || release.tag;
|
|
@@ -402,10 +343,6 @@ function formatRelease(release, packageName) {
|
|
|
402
343
|
fm.push("---");
|
|
403
344
|
return `${fm.join("\n")}\n\n# ${release.name || release.tag}\n\n${release.markdown}`;
|
|
404
345
|
}
|
|
405
|
-
/**
|
|
406
|
-
* Generate a unified summary index of all releases for quick LLM scanning.
|
|
407
|
-
* Includes GitHub releases, blog release posts, and CHANGELOG link.
|
|
408
|
-
*/
|
|
409
346
|
function generateReleaseIndex(releasesOrOpts, packageName) {
|
|
410
347
|
const opts = Array.isArray(releasesOrOpts) ? {
|
|
411
348
|
releases: releasesOrOpts,
|
|
@@ -447,18 +384,10 @@ function generateReleaseIndex(releasesOrOpts, packageName) {
|
|
|
447
384
|
}
|
|
448
385
|
return lines.join("\n");
|
|
449
386
|
}
|
|
450
|
-
/**
|
|
451
|
-
* Check if a single release is a stub redirecting to CHANGELOG.md.
|
|
452
|
-
* Short body (<500 chars) that mentions CHANGELOG indicates no real content.
|
|
453
|
-
*/
|
|
454
387
|
function isStubRelease(release) {
|
|
455
388
|
const body = (release.markdown || "").trim();
|
|
456
389
|
return body.length < 500 && /changelog\.md/i.test(body);
|
|
457
390
|
}
|
|
458
|
-
/**
|
|
459
|
-
* Fetch CHANGELOG.md from a GitHub repo at a specific ref as fallback.
|
|
460
|
-
* For monorepos, also checks packages/{shortName}/CHANGELOG.md.
|
|
461
|
-
*/
|
|
462
391
|
async function fetchChangelog(owner, repo, ref, packageName) {
|
|
463
392
|
const paths = [];
|
|
464
393
|
if (packageName) {
|
|
@@ -474,13 +403,6 @@ async function fetchChangelog(owner, repo, ref, packageName) {
|
|
|
474
403
|
}
|
|
475
404
|
return null;
|
|
476
405
|
}
|
|
477
|
-
/**
|
|
478
|
-
* Fetch release notes for a package. Returns CachedDoc[] with releases/{tag}.md files.
|
|
479
|
-
*
|
|
480
|
-
* Strategy:
|
|
481
|
-
* 1. Fetch GitHub releases, filter to package-specific tags for monorepos
|
|
482
|
-
* 2. If no releases found, try CHANGELOG.md as fallback
|
|
483
|
-
*/
|
|
484
406
|
async function fetchReleaseNotes(owner, repo, installedVersion, gitRef, packageName, fromDate, changelogRef) {
|
|
485
407
|
const selected = selectReleases(await fetchAllReleases(owner, repo), packageName, installedVersion, fromDate);
|
|
486
408
|
if (selected.length > 0) {
|
|
@@ -504,11 +426,6 @@ async function fetchReleaseNotes(owner, repo, installedVersion, gitRef, packageN
|
|
|
504
426
|
content: changelog
|
|
505
427
|
}];
|
|
506
428
|
}
|
|
507
|
-
//#endregion
|
|
508
|
-
//#region src/sources/blog-releases.ts
|
|
509
|
-
/**
|
|
510
|
-
* Format a blog release as markdown with YAML frontmatter
|
|
511
|
-
*/
|
|
512
429
|
function formatBlogRelease(release) {
|
|
513
430
|
return `${[
|
|
514
431
|
"---",
|
|
@@ -520,9 +437,6 @@ function formatBlogRelease(release) {
|
|
|
520
437
|
"---"
|
|
521
438
|
].join("\n")}\n\n# ${release.title}\n\n${release.markdown}`;
|
|
522
439
|
}
|
|
523
|
-
/**
|
|
524
|
-
* Fetch and parse a single blog post using preset metadata for version/date
|
|
525
|
-
*/
|
|
526
440
|
async function fetchBlogPost(entry) {
|
|
527
441
|
try {
|
|
528
442
|
const html = await $fetch(entry.url, {
|
|
@@ -550,11 +464,6 @@ async function fetchBlogPost(entry) {
|
|
|
550
464
|
return null;
|
|
551
465
|
}
|
|
552
466
|
}
|
|
553
|
-
/**
|
|
554
|
-
* Filter blog releases by installed version
|
|
555
|
-
* Only includes releases where version <= installedVersion
|
|
556
|
-
* Returns all releases if version parsing fails (fail-safe)
|
|
557
|
-
*/
|
|
558
467
|
function filterBlogsByVersion(entries, installedVersion) {
|
|
559
468
|
const installedSv = parseSemver(installedVersion);
|
|
560
469
|
if (!installedSv) return entries;
|
|
@@ -564,23 +473,13 @@ function filterBlogsByVersion(entries, installedVersion) {
|
|
|
564
473
|
return compareSemver(entrySv, installedSv) <= 0;
|
|
565
474
|
});
|
|
566
475
|
}
|
|
567
|
-
/**
|
|
568
|
-
* Fetch blog release notes from package presets
|
|
569
|
-
* Filters to only releases matching or older than the installed version
|
|
570
|
-
* Returns CachedDoc[] with releases/blog-{version}.md files
|
|
571
|
-
*/
|
|
572
476
|
async function fetchBlogReleases(packageName, installedVersion) {
|
|
573
477
|
const preset = getBlogPreset(packageName);
|
|
574
478
|
if (!preset) return [];
|
|
575
479
|
const filteredReleases = filterBlogsByVersion(preset.releases, installedVersion);
|
|
576
480
|
if (filteredReleases.length === 0) return [];
|
|
577
|
-
const
|
|
578
|
-
const
|
|
579
|
-
for (let i = 0; i < filteredReleases.length; i += batchSize) {
|
|
580
|
-
const batch = filteredReleases.slice(i, i + batchSize);
|
|
581
|
-
const results = await Promise.all(batch.map((entry) => fetchBlogPost(entry)));
|
|
582
|
-
for (const result of results) if (result) releases.push(result);
|
|
583
|
-
}
|
|
481
|
+
const limit = pLimit(3);
|
|
482
|
+
const releases = (await Promise.all(filteredReleases.map((entry) => limit(() => fetchBlogPost(entry))))).filter((r) => r !== null);
|
|
584
483
|
if (releases.length === 0) return [];
|
|
585
484
|
releases.sort((a, b) => {
|
|
586
485
|
const aVer = a.version.split(".").map(Number);
|
|
@@ -596,134 +495,12 @@ async function fetchBlogReleases(packageName, installedVersion) {
|
|
|
596
495
|
content: formatBlogRelease(r)
|
|
597
496
|
}));
|
|
598
497
|
}
|
|
599
|
-
//#endregion
|
|
600
|
-
//#region src/sources/crawl.ts
|
|
601
|
-
/**
|
|
602
|
-
* Website crawl doc source — fetches docs by crawling a URL pattern
|
|
603
|
-
*/
|
|
604
|
-
/**
|
|
605
|
-
* Crawl a URL pattern and return docs as cached doc format.
|
|
606
|
-
* Uses HTTP crawler (no browser needed) with sitemap discovery + glob filtering.
|
|
607
|
-
*
|
|
608
|
-
* @param url - URL with optional glob pattern (e.g. 'https://example.com/docs/**')
|
|
609
|
-
* @param onProgress - Optional progress callback
|
|
610
|
-
* @param maxPages - Max pages to crawl (default 200)
|
|
611
|
-
*/
|
|
612
|
-
async function fetchCrawledDocs(url, onProgress, maxPages = 200) {
|
|
613
|
-
const outputDir = join(tmpdir(), "skilld-crawl", Date.now().toString());
|
|
614
|
-
onProgress?.(`Crawling ${url}`);
|
|
615
|
-
const userLang = getUserLang();
|
|
616
|
-
const foreignUrls = /* @__PURE__ */ new Set();
|
|
617
|
-
const doCrawl = () => crawlAndGenerate({
|
|
618
|
-
urls: [url],
|
|
619
|
-
outputDir,
|
|
620
|
-
driver: "http",
|
|
621
|
-
generateLlmsTxt: false,
|
|
622
|
-
generateIndividualMd: true,
|
|
623
|
-
maxRequestsPerCrawl: maxPages,
|
|
624
|
-
onPage: (page) => {
|
|
625
|
-
const lang = extractHtmlLang(page.html);
|
|
626
|
-
if (lang && !lang.startsWith("en") && !lang.startsWith(userLang)) foreignUrls.add(page.url);
|
|
627
|
-
}
|
|
628
|
-
}, (progress) => {
|
|
629
|
-
if (progress.crawling.status === "processing" && progress.crawling.total > 0) onProgress?.(`Crawling ${progress.crawling.processed}/${progress.crawling.total} pages`);
|
|
630
|
-
});
|
|
631
|
-
let results = await doCrawl().catch((err) => {
|
|
632
|
-
onProgress?.(`Crawl failed: ${err?.message || err}`);
|
|
633
|
-
return [];
|
|
634
|
-
});
|
|
635
|
-
if (results.length === 0) {
|
|
636
|
-
onProgress?.("Retrying crawl");
|
|
637
|
-
results = await doCrawl().catch(() => []);
|
|
638
|
-
}
|
|
639
|
-
rmSync(outputDir, {
|
|
640
|
-
recursive: true,
|
|
641
|
-
force: true
|
|
642
|
-
});
|
|
643
|
-
const docs = [];
|
|
644
|
-
let localeFiltered = 0;
|
|
645
|
-
for (const result of results) {
|
|
646
|
-
if (!result.success || !result.content) continue;
|
|
647
|
-
if (foreignUrls.has(result.url)) {
|
|
648
|
-
localeFiltered++;
|
|
649
|
-
continue;
|
|
650
|
-
}
|
|
651
|
-
const segments = (new URL(result.url).pathname.replace(/\/$/, "") || "/index").split("/").filter(Boolean);
|
|
652
|
-
if (isForeignPathPrefix(segments[0], userLang)) {
|
|
653
|
-
localeFiltered++;
|
|
654
|
-
continue;
|
|
655
|
-
}
|
|
656
|
-
const path = `docs/${segments.join("/")}.md`;
|
|
657
|
-
docs.push({
|
|
658
|
-
path,
|
|
659
|
-
content: result.content
|
|
660
|
-
});
|
|
661
|
-
}
|
|
662
|
-
if (localeFiltered > 0) onProgress?.(`Filtered ${localeFiltered} foreign locale pages`);
|
|
663
|
-
onProgress?.(`Crawled ${docs.length} pages`);
|
|
664
|
-
return docs;
|
|
665
|
-
}
|
|
666
|
-
const HTML_LANG_RE = /<html[^>]*\slang=["']([^"']+)["']/i;
|
|
667
|
-
/** Extract lang attribute from <html> tag */
|
|
668
|
-
function extractHtmlLang(html) {
|
|
669
|
-
return HTML_LANG_RE.exec(html)?.[1]?.toLowerCase();
|
|
670
|
-
}
|
|
671
|
-
/** Common ISO 639-1 locale codes for i18n'd doc sites */
|
|
672
|
-
const LOCALE_CODES = new Set([
|
|
673
|
-
"ar",
|
|
674
|
-
"de",
|
|
675
|
-
"es",
|
|
676
|
-
"fr",
|
|
677
|
-
"id",
|
|
678
|
-
"it",
|
|
679
|
-
"ja",
|
|
680
|
-
"ko",
|
|
681
|
-
"nl",
|
|
682
|
-
"pl",
|
|
683
|
-
"pt",
|
|
684
|
-
"pt-br",
|
|
685
|
-
"ru",
|
|
686
|
-
"th",
|
|
687
|
-
"tr",
|
|
688
|
-
"uk",
|
|
689
|
-
"vi",
|
|
690
|
-
"zh",
|
|
691
|
-
"zh-cn",
|
|
692
|
-
"zh-tw"
|
|
693
|
-
]);
|
|
694
|
-
/** Check if a URL path segment is a known locale prefix foreign to both English and user's locale */
|
|
695
|
-
function isForeignPathPrefix(segment, userLang) {
|
|
696
|
-
if (!segment) return false;
|
|
697
|
-
const lower = segment.toLowerCase();
|
|
698
|
-
if (lower === "en" || lower.startsWith(userLang)) return false;
|
|
699
|
-
return LOCALE_CODES.has(lower);
|
|
700
|
-
}
|
|
701
|
-
/** Detect user's 2-letter language code from env (e.g. 'ja' from LANG=ja_JP.UTF-8) */
|
|
702
|
-
function getUserLang() {
|
|
703
|
-
const code = (process.env.LC_ALL || process.env.LANG || process.env.LANGUAGE || "").split(/[_.:-]/)[0]?.toLowerCase() || "";
|
|
704
|
-
return code.length >= 2 ? code.slice(0, 2) : "en";
|
|
705
|
-
}
|
|
706
|
-
/** Append glob pattern to a docs URL for crawling */
|
|
707
|
-
function toCrawlPattern(docsUrl) {
|
|
708
|
-
return `${docsUrl.replace(/\/+$/, "")}/**`;
|
|
709
|
-
}
|
|
710
|
-
//#endregion
|
|
711
|
-
//#region src/sources/issues.ts
|
|
712
|
-
/**
|
|
713
|
-
* GitHub issues fetching via gh CLI Search API
|
|
714
|
-
* Freshness-weighted scoring, type quotas, comment quality filtering
|
|
715
|
-
* Categorized by labels, noise filtered out, non-technical issues detected
|
|
716
|
-
*/
|
|
717
498
|
let _ghAvailable;
|
|
718
|
-
/**
|
|
719
|
-
* Check if gh CLI is installed and authenticated (cached)
|
|
720
|
-
*/
|
|
721
499
|
function isGhAvailable() {
|
|
722
500
|
if (_ghAvailable !== void 0) return _ghAvailable;
|
|
723
501
|
const { status } = spawnSync("gh", ["auth", "status"], { stdio: "ignore" });
|
|
724
502
|
return _ghAvailable = status === 0;
|
|
725
503
|
}
|
|
726
|
-
/** Labels that indicate noise — filter these out entirely */
|
|
727
504
|
const NOISE_LABELS = new Set([
|
|
728
505
|
"duplicate",
|
|
729
506
|
"stale",
|
|
@@ -735,7 +512,6 @@ const NOISE_LABELS = new Set([
|
|
|
735
512
|
"needs triage",
|
|
736
513
|
"triage"
|
|
737
514
|
]);
|
|
738
|
-
/** Labels that indicate feature requests — deprioritize */
|
|
739
515
|
const FEATURE_LABELS = new Set([
|
|
740
516
|
"enhancement",
|
|
741
517
|
"feature",
|
|
@@ -771,7 +547,6 @@ const DOCS_LABELS = new Set([
|
|
|
771
547
|
"doc",
|
|
772
548
|
"typo"
|
|
773
549
|
]);
|
|
774
|
-
/** Cache compiled word-boundary regexes per keyword set */
|
|
775
550
|
const labelRegexCache = /* @__PURE__ */ new WeakMap();
|
|
776
551
|
function getLabelRegex(keywords) {
|
|
777
552
|
let re = labelRegexCache.get(keywords);
|
|
@@ -782,18 +557,10 @@ function getLabelRegex(keywords) {
|
|
|
782
557
|
}
|
|
783
558
|
return re;
|
|
784
559
|
}
|
|
785
|
-
/**
|
|
786
|
-
* Check if a label matches any keyword from a set using word boundaries.
|
|
787
|
-
* Handles emoji-prefixed labels like ":sparkles: feature request" or ":lady_beetle: bug"
|
|
788
|
-
* without false positives on substrings (e.g. "debug" should not match "bug").
|
|
789
|
-
*/
|
|
790
560
|
function labelMatchesAny(label, keywords) {
|
|
791
561
|
if (keywords.has(label)) return true;
|
|
792
562
|
return getLabelRegex(keywords).test(label);
|
|
793
563
|
}
|
|
794
|
-
/**
|
|
795
|
-
* Classify an issue by its labels into a type useful for skill generation
|
|
796
|
-
*/
|
|
797
564
|
function classifyIssue(labels) {
|
|
798
565
|
const lower = labels.map((l) => l.toLowerCase());
|
|
799
566
|
if (lower.some((l) => labelMatchesAny(l, BUG_LABELS))) return "bug";
|
|
@@ -802,37 +569,20 @@ function classifyIssue(labels) {
|
|
|
802
569
|
if (lower.some((l) => labelMatchesAny(l, FEATURE_LABELS))) return "feature";
|
|
803
570
|
return "other";
|
|
804
571
|
}
|
|
805
|
-
/**
|
|
806
|
-
* Check if an issue should be filtered out entirely
|
|
807
|
-
*/
|
|
808
572
|
function isNoiseIssue(issue) {
|
|
809
573
|
if (issue.labels.map((l) => l.toLowerCase()).some((l) => labelMatchesAny(l, NOISE_LABELS))) return true;
|
|
810
574
|
if (issue.title.startsWith("☂️") || issue.title.startsWith("[META]") || issue.title.startsWith("[Tracking]")) return true;
|
|
811
575
|
return false;
|
|
812
576
|
}
|
|
813
|
-
/**
|
|
814
|
-
* Detect non-technical issues: fan mail, showcases, sentiment.
|
|
815
|
-
* Short body + no code + high reactions = likely non-technical.
|
|
816
|
-
* Note: roadmap/tracking issues are NOT filtered — they get score-boosted instead.
|
|
817
|
-
*/
|
|
818
577
|
function isNonTechnical(issue) {
|
|
819
578
|
const body = (issue.body || "").trim();
|
|
820
579
|
if (body.length < 200 && !hasCodeBlock(body) && issue.reactions > 50) return true;
|
|
821
580
|
if (/\b(?:love|thank|awesome|great work)\b/i.test(issue.title) && !hasCodeBlock(body)) return true;
|
|
822
581
|
return false;
|
|
823
582
|
}
|
|
824
|
-
/**
|
|
825
|
-
* Freshness-weighted score: reactions * decay(age_in_years)
|
|
826
|
-
* Steep decay so recent issues dominate over old high-reaction ones.
|
|
827
|
-
* At 0.6: 1yr=0.63x, 2yr=0.45x, 4yr=0.29x, 6yr=0.22x
|
|
828
|
-
*/
|
|
829
583
|
function freshnessScore(reactions, createdAt) {
|
|
830
584
|
return reactions * (1 / (1 + (Date.now() - new Date(createdAt).getTime()) / (365.25 * 24 * 60 * 60 * 1e3) * .6));
|
|
831
585
|
}
|
|
832
|
-
/**
|
|
833
|
-
* Type quotas — guarantee a mix of issue types.
|
|
834
|
-
* Bugs and questions get priority; feature requests are hard-capped.
|
|
835
|
-
*/
|
|
836
586
|
function applyTypeQuotas(issues, limit) {
|
|
837
587
|
const byType = /* @__PURE__ */ new Map();
|
|
838
588
|
for (const issue of issues) mapInsert(byType, issue.type, () => []).push(issue);
|
|
@@ -866,17 +616,11 @@ function applyTypeQuotas(issues, limit) {
|
|
|
866
616
|
}
|
|
867
617
|
return selected.sort((a, b) => b.score - a.score);
|
|
868
618
|
}
|
|
869
|
-
/**
|
|
870
|
-
* Body truncation limit based on reactions — high-reaction issues deserve more space
|
|
871
|
-
*/
|
|
872
619
|
function bodyLimit(reactions) {
|
|
873
620
|
if (reactions >= 10) return 2e3;
|
|
874
621
|
if (reactions >= 5) return 1500;
|
|
875
622
|
return 800;
|
|
876
623
|
}
|
|
877
|
-
/**
|
|
878
|
-
* Fetch issues for a state using GitHub Search API sorted by reactions
|
|
879
|
-
*/
|
|
880
624
|
function fetchIssuesByState(owner, repo, state, count, releasedAt, fromDate) {
|
|
881
625
|
const fetchCount = Math.min(count * 3, 100);
|
|
882
626
|
let datePart = "";
|
|
@@ -922,12 +666,6 @@ function oneYearAgo() {
|
|
|
922
666
|
d.setFullYear(d.getFullYear() - 1);
|
|
923
667
|
return isoDate(d.toISOString());
|
|
924
668
|
}
|
|
925
|
-
/**
|
|
926
|
-
* Batch-fetch top comments for issues via GraphQL.
|
|
927
|
-
* Enriches the top N highest-score issues with their best comments.
|
|
928
|
-
* Prioritizes: comments with code blocks, from maintainers, with high reactions.
|
|
929
|
-
* Filters out "+1", "any updates?", "same here" noise.
|
|
930
|
-
*/
|
|
931
669
|
function enrichWithComments(owner, repo, issues, topN = 15) {
|
|
932
670
|
const worth = issues.filter((i) => i.comments > 0 && (i.type === "bug" || i.type === "question" || i.reactions >= 3)).sort((a, b) => b.score - a.score).slice(0, topN);
|
|
933
671
|
if (worth.length === 0) return;
|
|
@@ -975,10 +713,6 @@ function enrichWithComments(owner, repo, issues, topN = 15) {
|
|
|
975
713
|
}
|
|
976
714
|
} catch {}
|
|
977
715
|
}
|
|
978
|
-
/**
|
|
979
|
-
* Try to detect which version fixed a closed issue from maintainer comments.
|
|
980
|
-
* Looks for version patterns in maintainer/collaborator comments.
|
|
981
|
-
*/
|
|
982
716
|
function detectResolvedVersion(comments) {
|
|
983
717
|
const maintainerComments = comments.filter((c) => c.isMaintainer);
|
|
984
718
|
for (const c of maintainerComments.reverse()) {
|
|
@@ -990,11 +724,6 @@ function detectResolvedVersion(comments) {
|
|
|
990
724
|
}
|
|
991
725
|
}
|
|
992
726
|
}
|
|
993
|
-
/**
|
|
994
|
-
* Fetch issues from a GitHub repo with freshness-weighted scoring and type quotas.
|
|
995
|
-
* Returns a balanced mix: bugs > questions > docs > other > features.
|
|
996
|
-
* Filters noise, non-technical content, and enriches with quality comments.
|
|
997
|
-
*/
|
|
998
727
|
async function fetchGitHubIssues(owner, repo, limit = 30, releasedAt, fromDate) {
|
|
999
728
|
if (!isGhAvailable()) return [];
|
|
1000
729
|
const openCount = Math.ceil(limit * .75);
|
|
@@ -1009,9 +738,6 @@ async function fetchGitHubIssues(owner, repo, limit = 30, releasedAt, fromDate)
|
|
|
1009
738
|
return [];
|
|
1010
739
|
}
|
|
1011
740
|
}
|
|
1012
|
-
/**
|
|
1013
|
-
* Format a single issue as markdown with YAML frontmatter
|
|
1014
|
-
*/
|
|
1015
741
|
function formatIssueAsMarkdown(issue) {
|
|
1016
742
|
const limit = bodyLimit(issue.reactions);
|
|
1017
743
|
const fmFields = {
|
|
@@ -1046,10 +772,6 @@ function formatIssueAsMarkdown(issue) {
|
|
|
1046
772
|
}
|
|
1047
773
|
return lines.join("\n");
|
|
1048
774
|
}
|
|
1049
|
-
/**
|
|
1050
|
-
* Generate a summary index of all issues for quick LLM scanning.
|
|
1051
|
-
* Groups by type so the LLM can quickly find bugs vs questions.
|
|
1052
|
-
*/
|
|
1053
775
|
function generateIssueIndex(issues) {
|
|
1054
776
|
const byType = /* @__PURE__ */ new Map();
|
|
1055
777
|
for (const issue of issues) mapInsert(byType, issue.type, () => []).push(issue);
|
|
@@ -1094,1173 +816,1236 @@ function generateIssueIndex(issues) {
|
|
|
1094
816
|
}
|
|
1095
817
|
return sections.join("\n");
|
|
1096
818
|
}
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
* Prioritizes Q&A and Help categories, includes accepted answers
|
|
1102
|
-
* Comment quality filtering, smart truncation, noise removal
|
|
1103
|
-
*/
|
|
1104
|
-
/** Categories most useful for skill generation (in priority order) */
|
|
1105
|
-
const HIGH_VALUE_CATEGORIES = new Set([
|
|
1106
|
-
"q&a",
|
|
1107
|
-
"help",
|
|
1108
|
-
"troubleshooting",
|
|
1109
|
-
"support"
|
|
1110
|
-
]);
|
|
1111
|
-
const LOW_VALUE_CATEGORIES = new Set([
|
|
1112
|
-
"show and tell",
|
|
1113
|
-
"ideas",
|
|
1114
|
-
"polls"
|
|
1115
|
-
]);
|
|
1116
|
-
/** Off-topic or spam title patterns — instant reject */
|
|
1117
|
-
const TITLE_NOISE_RE = /looking .*(?:developer|engineer|freelanc)|hiring|job post|guide me to (?:complete|finish|build)|help me (?:complete|finish|build)|seeking .* tutorial|recommend.* course/i;
|
|
1118
|
-
/** Minimum score for a discussion to be included */
|
|
1119
|
-
const MIN_DISCUSSION_SCORE = 3;
|
|
1120
|
-
/**
|
|
1121
|
-
* Score a comment for quality. Higher = more useful for skill generation.
|
|
1122
|
-
* Maintainers 3x, code blocks 2x, reactions linear.
|
|
1123
|
-
*/
|
|
1124
|
-
function scoreComment(c) {
|
|
1125
|
-
return (c.isMaintainer ? 3 : 1) * (hasCodeBlock(c.body) ? 2 : 1) * (1 + c.reactions);
|
|
819
|
+
async function fetchLlmsUrl(docsUrl) {
|
|
820
|
+
const llmsUrl = `${new URL(docsUrl).origin}/llms.txt`;
|
|
821
|
+
if (await verifyUrl(llmsUrl)) return llmsUrl;
|
|
822
|
+
return null;
|
|
1126
823
|
}
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
if (d.isMaintainer) score += 3;
|
|
1135
|
-
if (hasCodeBlock([
|
|
1136
|
-
d.body,
|
|
1137
|
-
d.answer || "",
|
|
1138
|
-
...d.topComments.map((c) => c.body)
|
|
1139
|
-
].join("\n"))) score += 3;
|
|
1140
|
-
score += Math.min(d.upvoteCount, 5);
|
|
1141
|
-
if (d.answer) {
|
|
1142
|
-
score += 2;
|
|
1143
|
-
if (d.answer.length > 100) score += 1;
|
|
1144
|
-
}
|
|
1145
|
-
if (d.topComments.some((c) => c.isMaintainer)) score += 2;
|
|
1146
|
-
if (d.topComments.some((c) => c.reactions > 0)) score += 1;
|
|
1147
|
-
return score;
|
|
824
|
+
async function fetchLlmsTxt(url) {
|
|
825
|
+
const content = await fetchText(url);
|
|
826
|
+
if (!content || content.length < 50) return null;
|
|
827
|
+
return {
|
|
828
|
+
raw: content,
|
|
829
|
+
links: parseMarkdownLinks(content)
|
|
830
|
+
};
|
|
1148
831
|
}
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
*/
|
|
1154
|
-
async function fetchGitHubDiscussions(owner, repo, limit = 20, releasedAt, fromDate) {
|
|
1155
|
-
if (!isGhAvailable()) return [];
|
|
1156
|
-
if (!fromDate && releasedAt) {
|
|
1157
|
-
const cutoff = new Date(releasedAt);
|
|
1158
|
-
cutoff.setMonth(cutoff.getMonth() + 6);
|
|
1159
|
-
if (cutoff < /* @__PURE__ */ new Date()) return [];
|
|
1160
|
-
}
|
|
832
|
+
function parseMarkdownLinks(content) {
|
|
833
|
+
return extractLinks(content).filter((l) => l.url.endsWith(".md"));
|
|
834
|
+
}
|
|
835
|
+
function isSafeUrl(url) {
|
|
1161
836
|
try {
|
|
1162
|
-
const
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
"-f",
|
|
1170
|
-
`repo=${repo}`
|
|
1171
|
-
], {
|
|
1172
|
-
encoding: "utf-8",
|
|
1173
|
-
maxBuffer: 10 * 1024 * 1024
|
|
1174
|
-
});
|
|
1175
|
-
if (!result) return [];
|
|
1176
|
-
const nodes = JSON.parse(result)?.data?.repository?.discussions?.nodes;
|
|
1177
|
-
if (!Array.isArray(nodes)) return [];
|
|
1178
|
-
const fromTs = fromDate ? new Date(fromDate).getTime() : null;
|
|
1179
|
-
return nodes.filter((d) => d.author && !BOT_USERS.has(d.author.login)).filter((d) => {
|
|
1180
|
-
const cat = (d.category?.name || "").toLowerCase();
|
|
1181
|
-
return !LOW_VALUE_CATEGORIES.has(cat);
|
|
1182
|
-
}).filter((d) => !fromTs || new Date(d.createdAt).getTime() >= fromTs).map((d) => {
|
|
1183
|
-
let answer;
|
|
1184
|
-
if (d.answer?.body) {
|
|
1185
|
-
const isMaintainer = [
|
|
1186
|
-
"OWNER",
|
|
1187
|
-
"MEMBER",
|
|
1188
|
-
"COLLABORATOR"
|
|
1189
|
-
].includes(d.answer.authorAssociation);
|
|
1190
|
-
const author = d.answer.author?.login;
|
|
1191
|
-
answer = `${isMaintainer && author ? `**@${author}** [maintainer]:\n\n` : ""}${d.answer.body}`;
|
|
1192
|
-
}
|
|
1193
|
-
const comments = (d.comments?.nodes || []).filter((c) => c.author && !BOT_USERS.has(c.author.login)).filter((c) => !COMMENT_NOISE_RE.test((c.body || "").trim())).map((c) => {
|
|
1194
|
-
const isMaintainer = [
|
|
1195
|
-
"OWNER",
|
|
1196
|
-
"MEMBER",
|
|
1197
|
-
"COLLABORATOR"
|
|
1198
|
-
].includes(c.authorAssociation);
|
|
1199
|
-
return {
|
|
1200
|
-
body: c.body || "",
|
|
1201
|
-
author: c.author.login,
|
|
1202
|
-
reactions: c.reactions?.totalCount || 0,
|
|
1203
|
-
isMaintainer
|
|
1204
|
-
};
|
|
1205
|
-
}).sort((a, b) => scoreComment(b) - scoreComment(a)).slice(0, 3);
|
|
1206
|
-
return {
|
|
1207
|
-
number: d.number,
|
|
1208
|
-
title: d.title,
|
|
1209
|
-
body: d.body || "",
|
|
1210
|
-
category: d.category?.name || "",
|
|
1211
|
-
createdAt: d.createdAt,
|
|
1212
|
-
url: d.url,
|
|
1213
|
-
upvoteCount: d.upvoteCount || 0,
|
|
1214
|
-
comments: d.comments?.totalCount || 0,
|
|
1215
|
-
isMaintainer: [
|
|
1216
|
-
"OWNER",
|
|
1217
|
-
"MEMBER",
|
|
1218
|
-
"COLLABORATOR"
|
|
1219
|
-
].includes(d.authorAssociation),
|
|
1220
|
-
answer,
|
|
1221
|
-
topComments: comments
|
|
1222
|
-
};
|
|
1223
|
-
}).map((d) => ({
|
|
1224
|
-
d,
|
|
1225
|
-
score: scoreDiscussion(d)
|
|
1226
|
-
})).filter(({ score }) => score >= MIN_DISCUSSION_SCORE).sort((a, b) => {
|
|
1227
|
-
const aHigh = HIGH_VALUE_CATEGORIES.has(a.d.category.toLowerCase()) ? 1 : 0;
|
|
1228
|
-
const bHigh = HIGH_VALUE_CATEGORIES.has(b.d.category.toLowerCase()) ? 1 : 0;
|
|
1229
|
-
if (aHigh !== bHigh) return bHigh - aHigh;
|
|
1230
|
-
return b.score - a.score;
|
|
1231
|
-
}).slice(0, limit).map(({ d }) => d);
|
|
837
|
+
const parsed = new URL(url);
|
|
838
|
+
if (parsed.protocol !== "https:") return false;
|
|
839
|
+
const host = parsed.hostname;
|
|
840
|
+
if (host === "localhost" || host === "0.0.0.0" || host === "[::1]") return false;
|
|
841
|
+
if (/^(?:127\.|10\.|172\.(?:1[6-9]|2\d|3[01])\.|192\.168\.|169\.254\.)/.test(host)) return false;
|
|
842
|
+
if (/^\[(?:f[cd]|fe[89ab]|::ffff:)/i.test(host)) return false;
|
|
843
|
+
return true;
|
|
1232
844
|
} catch {
|
|
1233
|
-
return
|
|
845
|
+
return false;
|
|
1234
846
|
}
|
|
1235
847
|
}
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
const lines = [
|
|
1252
|
-
fm,
|
|
1253
|
-
"",
|
|
1254
|
-
`# ${d.title}`
|
|
1255
|
-
];
|
|
1256
|
-
if (d.body) lines.push("", truncateBody(d.body, bodyLimit));
|
|
1257
|
-
if (d.answer) lines.push("", "---", "", "## Accepted Answer", "", truncateBody(d.answer, 1e3));
|
|
1258
|
-
else if (d.topComments.length > 0) {
|
|
1259
|
-
lines.push("", "---", "", "## Top Comments");
|
|
1260
|
-
for (const c of d.topComments) {
|
|
1261
|
-
const reactions = c.reactions > 0 ? ` (+${c.reactions})` : "";
|
|
1262
|
-
const maintainer = c.isMaintainer ? " [maintainer]" : "";
|
|
1263
|
-
lines.push("", `**@${c.author}**${maintainer}${reactions}:`, "", truncateBody(c.body, 600));
|
|
1264
|
-
}
|
|
1265
|
-
}
|
|
1266
|
-
return lines.join("\n");
|
|
848
|
+
async function downloadLlmsDocs(llmsContent, baseUrl, onProgress) {
|
|
849
|
+
const limit = pLimit(5);
|
|
850
|
+
let completed = 0;
|
|
851
|
+
return (await Promise.all(llmsContent.links.map((link) => limit(async () => {
|
|
852
|
+
const url = link.url.startsWith("http") ? link.url : `${baseUrl.replace(/\/$/, "")}${link.url.startsWith("/") ? "" : "/"}${link.url}`;
|
|
853
|
+
if (!isSafeUrl(url)) return null;
|
|
854
|
+
const content = await fetchText(url);
|
|
855
|
+
onProgress?.(link.url, ++completed, llmsContent.links.length);
|
|
856
|
+
if (content && content.length > 100) return {
|
|
857
|
+
url: link.url.startsWith("http") ? new URL(link.url).pathname : link.url,
|
|
858
|
+
title: link.title,
|
|
859
|
+
content
|
|
860
|
+
};
|
|
861
|
+
return null;
|
|
862
|
+
})))).filter((d) => d !== null);
|
|
1267
863
|
}
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
const byCategory = /* @__PURE__ */ new Map();
|
|
1274
|
-
for (const d of discussions) mapInsert(byCategory, d.category || "Uncategorized", () => []).push(d);
|
|
1275
|
-
const answered = discussions.filter((d) => d.answer).length;
|
|
1276
|
-
const sections = [
|
|
1277
|
-
[
|
|
1278
|
-
"---",
|
|
1279
|
-
`total: ${discussions.length}`,
|
|
1280
|
-
`answered: ${answered}`,
|
|
1281
|
-
"---"
|
|
1282
|
-
].join("\n"),
|
|
1283
|
-
"",
|
|
1284
|
-
"# Discussions Index",
|
|
1285
|
-
""
|
|
1286
|
-
];
|
|
1287
|
-
const cats = [...byCategory.keys()].sort((a, b) => {
|
|
1288
|
-
return (HIGH_VALUE_CATEGORIES.has(a.toLowerCase()) ? 0 : 1) - (HIGH_VALUE_CATEGORIES.has(b.toLowerCase()) ? 0 : 1) || a.localeCompare(b);
|
|
1289
|
-
});
|
|
1290
|
-
for (const cat of cats) {
|
|
1291
|
-
const group = byCategory.get(cat);
|
|
1292
|
-
sections.push(`## ${cat} (${group.length})`, "");
|
|
1293
|
-
for (const d of group) {
|
|
1294
|
-
const upvotes = d.upvoteCount > 0 ? ` (+${d.upvoteCount})` : "";
|
|
1295
|
-
const answered = d.answer ? " [answered]" : "";
|
|
1296
|
-
const date = isoDate(d.createdAt);
|
|
1297
|
-
sections.push(`- [#${d.number}](./discussion-${d.number}.md): ${d.title}${upvotes}${answered} (${date})`);
|
|
1298
|
-
}
|
|
1299
|
-
sections.push("");
|
|
864
|
+
function normalizeLlmsLinks(content, baseUrl) {
|
|
865
|
+
let normalized = content;
|
|
866
|
+
if (baseUrl) {
|
|
867
|
+
const escaped = baseUrl.replace(/\/$/, "").replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
868
|
+
normalized = normalized.replace(new RegExp(`\\]\\(${escaped}(/[^)]+\\.md)\\)`, "g"), "](./docs$1)");
|
|
1300
869
|
}
|
|
1301
|
-
|
|
870
|
+
normalized = normalized.replace(/\]\(\/([^)]+\.md)\)/g, "](./docs/$1)");
|
|
871
|
+
return normalized;
|
|
1302
872
|
}
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
function generateDocsIndex(docs) {
|
|
1314
|
-
const docFiles = docs.filter((d) => d.path.startsWith("docs/") && d.path.endsWith(".md") && !d.path.endsWith("_INDEX.md")).sort((a, b) => a.path.localeCompare(b.path));
|
|
1315
|
-
if (docFiles.length === 0) return "";
|
|
1316
|
-
const rootFiles = [];
|
|
1317
|
-
const byDir = /* @__PURE__ */ new Map();
|
|
1318
|
-
for (const doc of docFiles) {
|
|
1319
|
-
const rel = doc.path.slice(5);
|
|
1320
|
-
const dir = rel.includes("/") ? rel.slice(0, rel.lastIndexOf("/")) : "";
|
|
1321
|
-
if (!dir) rootFiles.push(doc);
|
|
1322
|
-
else {
|
|
1323
|
-
const list = byDir.get(dir);
|
|
1324
|
-
if (list) list.push(doc);
|
|
1325
|
-
else byDir.set(dir, [doc]);
|
|
873
|
+
function extractSections(content, patterns) {
|
|
874
|
+
const sections = [];
|
|
875
|
+
const parts = content.split(/\n---\n/);
|
|
876
|
+
for (const part of parts) {
|
|
877
|
+
const urlMatch = part.match(/^url: *(\S.*)$/m);
|
|
878
|
+
if (!urlMatch) continue;
|
|
879
|
+
const url = urlMatch[1];
|
|
880
|
+
if (patterns.some((p) => url.includes(p))) {
|
|
881
|
+
const contentStart = part.indexOf("\n", part.indexOf("url:"));
|
|
882
|
+
if (contentStart > -1) sections.push(part.slice(contentStart + 1));
|
|
1326
883
|
}
|
|
1327
884
|
}
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
const rel = file.path.slice(5);
|
|
1338
|
-
const title = extractTitle(file.content) || rel.replace(/\.md$/, "");
|
|
1339
|
-
const desc = extractDescription(file.content);
|
|
1340
|
-
const descPart = desc ? `: ${desc}` : "";
|
|
1341
|
-
sections.push(`- [${title}](./${rel})${descPart}`);
|
|
885
|
+
if (sections.length === 0) return null;
|
|
886
|
+
return sections.join("\n\n---\n\n");
|
|
887
|
+
}
|
|
888
|
+
const MIN_GIT_DOCS = 5;
|
|
889
|
+
const isShallowGitDocs = (n) => n > 0 && n < 5;
|
|
890
|
+
async function listFilesAtRef(owner, repo, ref) {
|
|
891
|
+
if (!isKnownPrivateRepo(owner, repo)) {
|
|
892
|
+
const data = await $fetch(`https://ungh.cc/repos/${owner}/${repo}/files/${ref}`).catch(() => null);
|
|
893
|
+
if (data?.files?.length) return data.files.map((f) => f.path);
|
|
1342
894
|
}
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
const rel = file.path.slice(5);
|
|
1348
|
-
const title = extractTitle(file.content) || rel.replace(/\.md$/, "").split("/").pop();
|
|
1349
|
-
const desc = extractDescription(file.content);
|
|
1350
|
-
const descPart = desc ? `: ${desc}` : "";
|
|
1351
|
-
sections.push(`- [${title}](./${rel})${descPart}`);
|
|
1352
|
-
}
|
|
1353
|
-
sections.push("");
|
|
895
|
+
const tree = await ghApi(`repos/${owner}/${repo}/git/trees/${ref}?recursive=1`);
|
|
896
|
+
if (tree?.tree?.length) {
|
|
897
|
+
markRepoPrivate(owner, repo);
|
|
898
|
+
return tree.tree.map((f) => f.path);
|
|
1354
899
|
}
|
|
1355
|
-
return
|
|
900
|
+
return [];
|
|
1356
901
|
}
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
"i18n",
|
|
1376
|
-
".git"
|
|
1377
|
-
];
|
|
1378
|
-
const SKIP_PATTERNS = [
|
|
1379
|
-
"*.min.*",
|
|
1380
|
-
"*.prod.*",
|
|
1381
|
-
"*.global.*",
|
|
1382
|
-
"*.browser.*",
|
|
1383
|
-
"*.map",
|
|
1384
|
-
"*.map.js",
|
|
1385
|
-
"CHANGELOG*",
|
|
1386
|
-
"LICENSE*",
|
|
1387
|
-
"README*"
|
|
1388
|
-
];
|
|
1389
|
-
const MAX_FILE_SIZE = 500 * 1024;
|
|
1390
|
-
/**
|
|
1391
|
-
* Glob .d.ts type definition files from a package directory, skipping junk.
|
|
1392
|
-
*/
|
|
1393
|
-
async function resolveEntryFiles(packageDir) {
|
|
1394
|
-
if (!existsSync(join(packageDir, "package.json"))) return [];
|
|
1395
|
-
const files = await glob(["**/*.d.{ts,mts,cts}"], {
|
|
1396
|
-
cwd: packageDir,
|
|
1397
|
-
ignore: [...SKIP_DIRS.map((d) => `**/${d}/**`), ...SKIP_PATTERNS],
|
|
1398
|
-
absolute: false,
|
|
1399
|
-
expandDirectories: false
|
|
1400
|
-
});
|
|
1401
|
-
const entries = [];
|
|
1402
|
-
for (const file of files) {
|
|
1403
|
-
const absPath = join(packageDir, file);
|
|
1404
|
-
let content;
|
|
1405
|
-
try {
|
|
1406
|
-
content = readFileSync(absPath, "utf-8");
|
|
1407
|
-
} catch {
|
|
1408
|
-
continue;
|
|
902
|
+
async function findGitTag(owner, repo, version, packageName, branchHint) {
|
|
903
|
+
const candidates = [`v${version}`, version];
|
|
904
|
+
if (packageName) candidates.push(`${packageName}@${version}`);
|
|
905
|
+
for (const tag of candidates) {
|
|
906
|
+
const files = await listFilesAtRef(owner, repo, tag);
|
|
907
|
+
if (files.length > 0) return {
|
|
908
|
+
ref: tag,
|
|
909
|
+
files
|
|
910
|
+
};
|
|
911
|
+
}
|
|
912
|
+
if (packageName) {
|
|
913
|
+
const latestTag = await findLatestReleaseTag(owner, repo, packageName);
|
|
914
|
+
if (latestTag) {
|
|
915
|
+
const files = await listFilesAtRef(owner, repo, latestTag);
|
|
916
|
+
if (files.length > 0) return {
|
|
917
|
+
ref: latestTag,
|
|
918
|
+
files
|
|
919
|
+
};
|
|
1409
920
|
}
|
|
1410
|
-
if (content.length > MAX_FILE_SIZE) continue;
|
|
1411
|
-
entries.push({
|
|
1412
|
-
path: file,
|
|
1413
|
-
content,
|
|
1414
|
-
type: "types"
|
|
1415
|
-
});
|
|
1416
921
|
}
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
* Supports GitHub shorthand (owner/repo), full URLs, SSH, GitLab, and local paths.
|
|
1425
|
-
* Skills are pre-authored SKILL.md files — no doc resolution or LLM generation needed.
|
|
1426
|
-
*/
|
|
1427
|
-
/**
|
|
1428
|
-
* Detect whether an input string is a git skill source.
|
|
1429
|
-
* Returns null for npm package names (including scoped @scope/pkg).
|
|
1430
|
-
*/
|
|
1431
|
-
function parseGitSkillInput(input) {
|
|
1432
|
-
const trimmed = input.trim();
|
|
1433
|
-
if (trimmed.startsWith("@")) return null;
|
|
1434
|
-
if (trimmed.startsWith("./") || trimmed.startsWith("../") || trimmed.startsWith("/") || trimmed.startsWith("~")) return {
|
|
1435
|
-
type: "local",
|
|
1436
|
-
localPath: trimmed.startsWith("~") ? resolve(process.env.HOME || "", trimmed.slice(1)) : resolve(trimmed)
|
|
1437
|
-
};
|
|
1438
|
-
if (trimmed.startsWith("git@")) {
|
|
1439
|
-
const gh = parseGitHubUrl(normalizeRepoUrl(trimmed));
|
|
1440
|
-
if (gh) return {
|
|
1441
|
-
type: "github",
|
|
1442
|
-
owner: gh.owner,
|
|
1443
|
-
repo: gh.repo
|
|
922
|
+
const branches = branchHint ? [branchHint, ...["main", "master"].filter((b) => b !== branchHint)] : ["main", "master"];
|
|
923
|
+
for (const branch of branches) {
|
|
924
|
+
const files = await listFilesAtRef(owner, repo, branch);
|
|
925
|
+
if (files.length > 0) return {
|
|
926
|
+
ref: branch,
|
|
927
|
+
files,
|
|
928
|
+
fallback: true
|
|
1444
929
|
};
|
|
1445
|
-
return null;
|
|
1446
930
|
}
|
|
1447
|
-
if (trimmed.startsWith("https://") || trimmed.startsWith("http://")) return parseGitUrl(trimmed);
|
|
1448
|
-
if (/^[\w.-]+\/[\w.-]+$/.test(trimmed)) return {
|
|
1449
|
-
type: "github",
|
|
1450
|
-
owner: trimmed.split("/")[0],
|
|
1451
|
-
repo: trimmed.split("/")[1]
|
|
1452
|
-
};
|
|
1453
931
|
return null;
|
|
1454
932
|
}
|
|
1455
|
-
function
|
|
1456
|
-
|
|
1457
|
-
const
|
|
1458
|
-
if (
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
ref: parts[3],
|
|
1468
|
-
skillPath: parts.length > 4 ? parts.slice(4).join("/") : void 0
|
|
1469
|
-
};
|
|
1470
|
-
return {
|
|
1471
|
-
type: "github",
|
|
1472
|
-
owner,
|
|
1473
|
-
repo
|
|
1474
|
-
};
|
|
1475
|
-
}
|
|
1476
|
-
if (parsed.hostname === "gitlab.com") {
|
|
1477
|
-
const parts = parsed.pathname.replace(/^\//, "").replace(/\.git$/, "").split("/");
|
|
1478
|
-
const owner = parts[0];
|
|
1479
|
-
const repo = parts[1];
|
|
1480
|
-
if (!owner || !repo) return null;
|
|
1481
|
-
return {
|
|
1482
|
-
type: "gitlab",
|
|
1483
|
-
owner,
|
|
1484
|
-
repo
|
|
1485
|
-
};
|
|
1486
|
-
}
|
|
1487
|
-
return null;
|
|
1488
|
-
} catch {
|
|
1489
|
-
return null;
|
|
933
|
+
async function fetchUnghReleases(owner, repo) {
|
|
934
|
+
if (!isKnownPrivateRepo(owner, repo)) {
|
|
935
|
+
const data = await $fetch(`https://ungh.cc/repos/${owner}/${repo}/releases`).catch(() => null);
|
|
936
|
+
if (data?.releases?.length) return data.releases;
|
|
937
|
+
}
|
|
938
|
+
const raw = await ghApiPaginated(`repos/${owner}/${repo}/releases`);
|
|
939
|
+
if (raw.length > 0) {
|
|
940
|
+
markRepoPrivate(owner, repo);
|
|
941
|
+
return raw.map((r) => ({
|
|
942
|
+
tag: r.tag_name,
|
|
943
|
+
publishedAt: r.published_at
|
|
944
|
+
}));
|
|
1490
945
|
}
|
|
946
|
+
return [];
|
|
1491
947
|
}
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
function parseSkillFrontmatterName(content) {
|
|
1496
|
-
const fm = parseFrontmatter(content);
|
|
1497
|
-
return {
|
|
1498
|
-
name: fm.name,
|
|
1499
|
-
description: fm.description
|
|
1500
|
-
};
|
|
948
|
+
async function findLatestReleaseTag(owner, repo, packageName) {
|
|
949
|
+
const prefix = `${packageName}@`;
|
|
950
|
+
return (await fetchUnghReleases(owner, repo)).find((r) => r.tag.startsWith(prefix))?.tag ?? null;
|
|
1501
951
|
}
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
const files = [];
|
|
1505
|
-
if (!existsSync(dir)) return files;
|
|
1506
|
-
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
1507
|
-
const relPath = prefix ? `${prefix}/${entry.name}` : entry.name;
|
|
1508
|
-
const fullPath = resolve(dir, entry.name);
|
|
1509
|
-
if (entry.isDirectory()) files.push(...collectFiles(fullPath, relPath));
|
|
1510
|
-
else if (entry.isFile()) files.push({
|
|
1511
|
-
path: relPath,
|
|
1512
|
-
content: readFileSync(fullPath, "utf-8")
|
|
1513
|
-
});
|
|
1514
|
-
}
|
|
1515
|
-
return files;
|
|
952
|
+
function filterDocFiles(files, pathPrefix) {
|
|
953
|
+
return files.filter((f) => f.startsWith(pathPrefix) && /\.(?:md|mdx)$/.test(f));
|
|
1516
954
|
}
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
955
|
+
const FRAMEWORK_NAMES = new Set([
|
|
956
|
+
"vue",
|
|
957
|
+
"react",
|
|
958
|
+
"solid",
|
|
959
|
+
"angular",
|
|
960
|
+
"svelte",
|
|
961
|
+
"preact",
|
|
962
|
+
"lit",
|
|
963
|
+
"qwik"
|
|
964
|
+
]);
|
|
965
|
+
function filterFrameworkDocs(files, packageName) {
|
|
966
|
+
if (!packageName) return files;
|
|
967
|
+
const shortName = packageName.replace(/^@.*\//, "");
|
|
968
|
+
const targetFramework = [...FRAMEWORK_NAMES].find((fw) => shortName.includes(fw));
|
|
969
|
+
if (!targetFramework) return files;
|
|
970
|
+
const otherFrameworks = [...FRAMEWORK_NAMES].filter((fw) => fw !== targetFramework);
|
|
971
|
+
const excludePattern = new RegExp(`\\b(?:${otherFrameworks.join("|")})\\b`);
|
|
972
|
+
return files.filter((f) => !excludePattern.test(f));
|
|
1525
973
|
}
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
974
|
+
const NOISE_PATTERNS = [
|
|
975
|
+
/^\.changeset\//,
|
|
976
|
+
/CHANGELOG\.md$/i,
|
|
977
|
+
/CONTRIBUTING\.md$/i,
|
|
978
|
+
/^\.github\//
|
|
979
|
+
];
|
|
980
|
+
const EXCLUDE_DIRS = new Set([
|
|
981
|
+
"test",
|
|
982
|
+
"tests",
|
|
983
|
+
"__tests__",
|
|
984
|
+
"fixtures",
|
|
985
|
+
"fixture",
|
|
986
|
+
"examples",
|
|
987
|
+
"example",
|
|
988
|
+
"node_modules",
|
|
989
|
+
".git",
|
|
990
|
+
"dist",
|
|
991
|
+
"build",
|
|
992
|
+
"coverage",
|
|
993
|
+
"e2e",
|
|
994
|
+
"spec",
|
|
995
|
+
"mocks",
|
|
996
|
+
"__mocks__"
|
|
997
|
+
]);
|
|
998
|
+
const DOC_DIR_BONUS = new Set([
|
|
999
|
+
"docs",
|
|
1000
|
+
"documentation",
|
|
1001
|
+
"pages",
|
|
1002
|
+
"content",
|
|
1003
|
+
"website",
|
|
1004
|
+
"guide",
|
|
1005
|
+
"guides",
|
|
1006
|
+
"wiki",
|
|
1007
|
+
"manual",
|
|
1008
|
+
"api"
|
|
1009
|
+
]);
|
|
1010
|
+
function hasExcludedDir(path) {
|
|
1011
|
+
return path.split("/").some((p) => EXCLUDE_DIRS.has(p.toLowerCase()));
|
|
1012
|
+
}
|
|
1013
|
+
function getPathDepth(path) {
|
|
1014
|
+
return path.split("/").filter(Boolean).length;
|
|
1015
|
+
}
|
|
1016
|
+
function hasDocDirBonus(path) {
|
|
1017
|
+
return path.split("/").some((p) => DOC_DIR_BONUS.has(p.toLowerCase()));
|
|
1018
|
+
}
|
|
1019
|
+
function scoreDocDir(dir, fileCount) {
|
|
1020
|
+
const depth = getPathDepth(dir) || 1;
|
|
1021
|
+
return fileCount * (hasDocDirBonus(dir) ? 1.5 : 1) / depth;
|
|
1022
|
+
}
|
|
1023
|
+
function discoverDocFiles(allFiles, packageName) {
|
|
1024
|
+
const mdFiles = allFiles.filter((f) => /\.(?:md|mdx)$/.test(f)).filter((f) => !NOISE_PATTERNS.some((p) => p.test(f))).filter((f) => f.includes("/"));
|
|
1025
|
+
if (packageName?.includes("/")) {
|
|
1026
|
+
const subPkgPrefix = `packages/${packageName.split("/").pop().toLowerCase()}/`;
|
|
1027
|
+
const subPkgFiles = mdFiles.filter((f) => f.startsWith(subPkgPrefix));
|
|
1028
|
+
if (subPkgFiles.length >= 3) return {
|
|
1029
|
+
files: subPkgFiles,
|
|
1030
|
+
prefix: subPkgPrefix
|
|
1031
|
+
};
|
|
1535
1032
|
}
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1033
|
+
const docsGroups = /* @__PURE__ */ new Map();
|
|
1034
|
+
for (const file of mdFiles) {
|
|
1035
|
+
const docsIdx = file.lastIndexOf("/docs/");
|
|
1036
|
+
if (docsIdx === -1) continue;
|
|
1037
|
+
mapInsert(docsGroups, file.slice(0, docsIdx + 6), () => []).push(file);
|
|
1539
1038
|
}
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1039
|
+
if (docsGroups.size > 0) {
|
|
1040
|
+
const largest = [...docsGroups.entries()].sort((a, b) => b[1].length - a[1].length)[0];
|
|
1041
|
+
if (largest[1].length >= 3) {
|
|
1042
|
+
const fullPrefix = largest[0];
|
|
1043
|
+
const docsIdx = fullPrefix.lastIndexOf("docs/");
|
|
1044
|
+
const stripPrefix = docsIdx > 0 ? fullPrefix.slice(0, docsIdx) : "";
|
|
1045
|
+
return {
|
|
1046
|
+
files: largest[1],
|
|
1047
|
+
prefix: stripPrefix
|
|
1048
|
+
};
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
const dirGroups = /* @__PURE__ */ new Map();
|
|
1052
|
+
for (const file of mdFiles) {
|
|
1053
|
+
if (hasExcludedDir(file)) continue;
|
|
1054
|
+
const lastSlash = file.lastIndexOf("/");
|
|
1055
|
+
if (lastSlash === -1) continue;
|
|
1056
|
+
mapInsert(dirGroups, file.slice(0, lastSlash + 1), () => []).push(file);
|
|
1057
|
+
}
|
|
1058
|
+
if (dirGroups.size === 0) return null;
|
|
1059
|
+
const scored = Array.from(dirGroups.entries(), ([dir, files]) => ({
|
|
1060
|
+
dir,
|
|
1061
|
+
files,
|
|
1062
|
+
score: scoreDocDir(dir, files.length)
|
|
1063
|
+
})).filter((d) => d.files.length >= 5).sort((a, b) => b.score - a.score);
|
|
1064
|
+
if (scored.length === 0) return null;
|
|
1065
|
+
const best = scored[0];
|
|
1550
1066
|
return {
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
path: repoPath,
|
|
1554
|
-
content,
|
|
1555
|
-
files
|
|
1067
|
+
files: best.files,
|
|
1068
|
+
prefix: best.dir
|
|
1556
1069
|
};
|
|
1557
1070
|
}
|
|
1558
|
-
async function
|
|
1559
|
-
|
|
1560
|
-
if (!owner || !repo) return { skills: [] };
|
|
1561
|
-
const ref = source.ref || "main";
|
|
1562
|
-
const refs = ref === "main" ? ["main", "master"] : [ref];
|
|
1563
|
-
for (const tryRef of refs) {
|
|
1564
|
-
const skills = await downloadGitHubSkills(owner, repo, tryRef, source.skillPath, onProgress);
|
|
1565
|
-
if (skills.length > 0) return { skills };
|
|
1566
|
-
}
|
|
1567
|
-
return { skills: [] };
|
|
1071
|
+
async function listDocsAtRef(owner, repo, ref, pathPrefix = "docs/") {
|
|
1072
|
+
return filterDocFiles(await listFilesAtRef(owner, repo, ref), pathPrefix);
|
|
1568
1073
|
}
|
|
1569
|
-
async function
|
|
1570
|
-
const
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1074
|
+
async function fetchGitDocs(owner, repo, version, packageName, repoUrl) {
|
|
1075
|
+
const override = packageName ? getDocOverride(packageName) : void 0;
|
|
1076
|
+
if (override) {
|
|
1077
|
+
const ref = override.ref || "main";
|
|
1078
|
+
const fallback = !override.ref;
|
|
1079
|
+
const files = await listDocsAtRef(override.owner, override.repo, ref, `${override.path}/`);
|
|
1080
|
+
if (files.length === 0) return null;
|
|
1081
|
+
return {
|
|
1082
|
+
baseUrl: `https://raw.githubusercontent.com/${override.owner}/${override.repo}/${ref}`,
|
|
1083
|
+
ref,
|
|
1084
|
+
files,
|
|
1085
|
+
fallback,
|
|
1086
|
+
docsPrefix: `${override.path}/` !== "docs/" ? `${override.path}/` : void 0
|
|
1087
|
+
};
|
|
1088
|
+
}
|
|
1089
|
+
const tag = await findGitTag(owner, repo, version, packageName, repoUrl ? extractBranchHint(repoUrl) : void 0);
|
|
1090
|
+
if (!tag) return null;
|
|
1091
|
+
let docs = filterDocFiles(tag.files, "docs/");
|
|
1092
|
+
let docsPrefix;
|
|
1093
|
+
let allFiles;
|
|
1094
|
+
if (docs.length === 0) {
|
|
1095
|
+
const discovered = discoverDocFiles(tag.files, packageName);
|
|
1096
|
+
if (discovered) {
|
|
1097
|
+
docs = discovered.files;
|
|
1098
|
+
docsPrefix = discovered.prefix || void 0;
|
|
1099
|
+
allFiles = tag.files;
|
|
1581
1100
|
}
|
|
1582
|
-
onProgress?.(`Downloading ${owner}/${repo}/skills@${ref}`);
|
|
1583
|
-
try {
|
|
1584
|
-
const { dir } = await downloadTemplate(`github:${owner}/${repo}/skills#${ref}`, {
|
|
1585
|
-
dir: tempDir,
|
|
1586
|
-
force: true,
|
|
1587
|
-
auth: getGitHubToken() || void 0
|
|
1588
|
-
});
|
|
1589
|
-
const skills = [];
|
|
1590
|
-
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
1591
|
-
if (!entry.isDirectory()) continue;
|
|
1592
|
-
const skill = readLocalSkill(resolve(dir, entry.name), `skills/${entry.name}`);
|
|
1593
|
-
if (skill) skills.push(skill);
|
|
1594
|
-
}
|
|
1595
|
-
if (skills.length > 0) {
|
|
1596
|
-
onProgress?.(`Found ${skills.length} skill(s)`);
|
|
1597
|
-
return skills;
|
|
1598
|
-
}
|
|
1599
|
-
} catch {}
|
|
1600
|
-
const content = await fetchGitHubRaw(`https://raw.githubusercontent.com/${owner}/${repo}/${ref}/SKILL.md`);
|
|
1601
|
-
if (content) {
|
|
1602
|
-
const fm = parseSkillFrontmatterName(content);
|
|
1603
|
-
onProgress?.("Found 1 skill");
|
|
1604
|
-
return [{
|
|
1605
|
-
name: fm.name || repo,
|
|
1606
|
-
description: fm.description || "",
|
|
1607
|
-
path: "",
|
|
1608
|
-
content,
|
|
1609
|
-
files: []
|
|
1610
|
-
}];
|
|
1611
|
-
}
|
|
1612
|
-
return [];
|
|
1613
|
-
} catch {
|
|
1614
|
-
return [];
|
|
1615
|
-
} finally {
|
|
1616
|
-
rmSync(tempDir, {
|
|
1617
|
-
recursive: true,
|
|
1618
|
-
force: true
|
|
1619
|
-
});
|
|
1620
|
-
}
|
|
1621
|
-
}
|
|
1622
|
-
async function fetchGitLabSkills(source, onProgress) {
|
|
1623
|
-
const { owner, repo } = source;
|
|
1624
|
-
if (!owner || !repo) return { skills: [] };
|
|
1625
|
-
const ref = source.ref || "main";
|
|
1626
|
-
const tempDir = join(tmpdir(), `skilld-gitlab-${Date.now()}`);
|
|
1627
|
-
try {
|
|
1628
|
-
const subdir = source.skillPath || "skills";
|
|
1629
|
-
onProgress?.(`Downloading ${owner}/${repo}/${subdir}@${ref}`);
|
|
1630
|
-
const { dir } = await downloadTemplate(`gitlab:${owner}/${repo}/${subdir}#${ref}`, {
|
|
1631
|
-
dir: tempDir,
|
|
1632
|
-
force: true
|
|
1633
|
-
});
|
|
1634
|
-
if (source.skillPath) {
|
|
1635
|
-
const skill = readLocalSkill(dir, source.skillPath);
|
|
1636
|
-
return { skills: skill ? [skill] : [] };
|
|
1637
|
-
}
|
|
1638
|
-
const skills = [];
|
|
1639
|
-
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
1640
|
-
if (!entry.isDirectory()) continue;
|
|
1641
|
-
const skill = readLocalSkill(resolve(dir, entry.name), `skills/${entry.name}`);
|
|
1642
|
-
if (skill) skills.push(skill);
|
|
1643
|
-
}
|
|
1644
|
-
if (skills.length > 0) {
|
|
1645
|
-
onProgress?.(`Found ${skills.length} skill(s)`);
|
|
1646
|
-
return { skills };
|
|
1647
|
-
}
|
|
1648
|
-
const content = await $fetch(`https://gitlab.com/${owner}/${repo}/-/raw/${ref}/SKILL.md`, { responseType: "text" }).catch(() => null);
|
|
1649
|
-
if (content) {
|
|
1650
|
-
const fm = parseSkillFrontmatterName(content);
|
|
1651
|
-
return { skills: [{
|
|
1652
|
-
name: fm.name || repo,
|
|
1653
|
-
description: fm.description || "",
|
|
1654
|
-
path: "",
|
|
1655
|
-
content,
|
|
1656
|
-
files: []
|
|
1657
|
-
}] };
|
|
1658
|
-
}
|
|
1659
|
-
return { skills: [] };
|
|
1660
|
-
} catch {
|
|
1661
|
-
return { skills: [] };
|
|
1662
|
-
} finally {
|
|
1663
|
-
rmSync(tempDir, {
|
|
1664
|
-
recursive: true,
|
|
1665
|
-
force: true
|
|
1666
|
-
});
|
|
1667
1101
|
}
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
//#region src/sources/llms.ts
|
|
1671
|
-
/**
|
|
1672
|
-
* Check for llms.txt at a docs URL, returns the llms.txt URL if found
|
|
1673
|
-
*/
|
|
1674
|
-
async function fetchLlmsUrl(docsUrl) {
|
|
1675
|
-
const llmsUrl = `${new URL(docsUrl).origin}/llms.txt`;
|
|
1676
|
-
if (await verifyUrl(llmsUrl)) return llmsUrl;
|
|
1677
|
-
return null;
|
|
1678
|
-
}
|
|
1679
|
-
/**
|
|
1680
|
-
* Fetch and parse llms.txt content
|
|
1681
|
-
*/
|
|
1682
|
-
async function fetchLlmsTxt(url) {
|
|
1683
|
-
const content = await fetchText(url);
|
|
1684
|
-
if (!content || content.length < 50) return null;
|
|
1102
|
+
docs = filterFrameworkDocs(docs, packageName);
|
|
1103
|
+
if (docs.length === 0) return null;
|
|
1685
1104
|
return {
|
|
1686
|
-
|
|
1687
|
-
|
|
1105
|
+
baseUrl: `https://raw.githubusercontent.com/${owner}/${repo}/${tag.ref}`,
|
|
1106
|
+
ref: tag.ref,
|
|
1107
|
+
files: docs,
|
|
1108
|
+
docsPrefix,
|
|
1109
|
+
allFiles,
|
|
1110
|
+
fallback: tag.fallback
|
|
1688
1111
|
};
|
|
1689
1112
|
}
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
*/
|
|
1693
|
-
function parseMarkdownLinks(content) {
|
|
1694
|
-
return extractLinks(content).filter((l) => l.url.endsWith(".md"));
|
|
1113
|
+
function normalizePath(p) {
|
|
1114
|
+
return p.replace(/^\//, "").replace(/\.(?:md|mdx)$/, "");
|
|
1695
1115
|
}
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1116
|
+
function validateGitDocsWithLlms(llmsLinks, repoFiles) {
|
|
1117
|
+
if (llmsLinks.length === 0) return {
|
|
1118
|
+
isValid: true,
|
|
1119
|
+
matchRatio: 1
|
|
1120
|
+
};
|
|
1121
|
+
const sample = llmsLinks.slice(0, 10);
|
|
1122
|
+
const normalizedLinks = sample.map((link) => {
|
|
1123
|
+
let path = link.url;
|
|
1124
|
+
if (path.startsWith("http")) try {
|
|
1125
|
+
path = new URL(path).pathname;
|
|
1126
|
+
} catch {}
|
|
1127
|
+
return normalizePath(path);
|
|
1128
|
+
});
|
|
1129
|
+
const repoNormalized = new Set(repoFiles.map(normalizePath));
|
|
1130
|
+
let matches = 0;
|
|
1131
|
+
for (const linkPath of normalizedLinks) for (const repoPath of repoNormalized) if (repoPath === linkPath || repoPath.endsWith(`/${linkPath}`)) {
|
|
1132
|
+
matches++;
|
|
1133
|
+
break;
|
|
1711
1134
|
}
|
|
1135
|
+
const matchRatio = matches / sample.length;
|
|
1136
|
+
return {
|
|
1137
|
+
isValid: matchRatio >= .3,
|
|
1138
|
+
matchRatio
|
|
1139
|
+
};
|
|
1712
1140
|
}
|
|
1713
|
-
async function
|
|
1714
|
-
const
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
}
|
|
1726
|
-
return null;
|
|
1727
|
-
})))).filter((d) => d !== null);
|
|
1728
|
-
}
|
|
1729
|
-
/**
|
|
1730
|
-
* Normalize llms.txt links to relative paths for local access
|
|
1731
|
-
* Handles: absolute URLs, root-relative paths, and relative paths
|
|
1732
|
-
*/
|
|
1733
|
-
function normalizeLlmsLinks(content, baseUrl) {
|
|
1734
|
-
let normalized = content;
|
|
1735
|
-
if (baseUrl) {
|
|
1736
|
-
const escaped = baseUrl.replace(/\/$/, "").replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1737
|
-
normalized = normalized.replace(new RegExp(`\\]\\(${escaped}(/[^)]+\\.md)\\)`, "g"), "](./docs$1)");
|
|
1141
|
+
async function verifyNpmRepo(owner, repo, packageName) {
|
|
1142
|
+
const base = `https://raw.githubusercontent.com/${owner}/${repo}/HEAD`;
|
|
1143
|
+
const paths = [
|
|
1144
|
+
"package.json",
|
|
1145
|
+
`packages/${packageName.replace(/^@.*\//, "")}/package.json`,
|
|
1146
|
+
`packages/${packageName.replace(/^@/, "").replace("/", "-")}/package.json`
|
|
1147
|
+
];
|
|
1148
|
+
for (const path of paths) {
|
|
1149
|
+
const text = await fetchGitHubRaw(`${base}/${path}`);
|
|
1150
|
+
if (!text) continue;
|
|
1151
|
+
try {
|
|
1152
|
+
if (JSON.parse(text).name === packageName) return true;
|
|
1153
|
+
} catch {}
|
|
1738
1154
|
}
|
|
1739
|
-
|
|
1740
|
-
return normalized;
|
|
1155
|
+
return false;
|
|
1741
1156
|
}
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
const parts = content.split(/\n---\n/);
|
|
1749
|
-
for (const part of parts) {
|
|
1750
|
-
const urlMatch = part.match(/^url: *(\S.*)$/m);
|
|
1751
|
-
if (!urlMatch) continue;
|
|
1752
|
-
const url = urlMatch[1];
|
|
1753
|
-
if (patterns.some((p) => url.includes(p))) {
|
|
1754
|
-
const contentStart = part.indexOf("\n", part.indexOf("url:"));
|
|
1755
|
-
if (contentStart > -1) sections.push(part.slice(contentStart + 1));
|
|
1157
|
+
async function searchGitHubRepo(packageName) {
|
|
1158
|
+
const shortName = packageName.replace(/^@.*\//, "");
|
|
1159
|
+
for (const candidate of [packageName.replace(/^@/, "").replace("/", "/"), shortName]) {
|
|
1160
|
+
if (!candidate.includes("/")) {
|
|
1161
|
+
if ((await $fetch.raw(`https://ungh.cc/repos/${shortName}/${shortName}`).catch(() => null))?.ok) return `https://github.com/${shortName}/${shortName}`;
|
|
1162
|
+
continue;
|
|
1756
1163
|
}
|
|
1164
|
+
if ((await $fetch.raw(`https://ungh.cc/repos/${candidate}`).catch(() => null))?.ok) return `https://github.com/${candidate}`;
|
|
1757
1165
|
}
|
|
1758
|
-
|
|
1759
|
-
|
|
1166
|
+
const searchTerm = packageName.replace(/^@/, "");
|
|
1167
|
+
if (isGhAvailable()) try {
|
|
1168
|
+
const { stdout: json } = spawnSync("gh", [
|
|
1169
|
+
"search",
|
|
1170
|
+
"repos",
|
|
1171
|
+
searchTerm,
|
|
1172
|
+
"--json",
|
|
1173
|
+
"fullName",
|
|
1174
|
+
"--limit",
|
|
1175
|
+
"5"
|
|
1176
|
+
], {
|
|
1177
|
+
encoding: "utf-8",
|
|
1178
|
+
timeout: 15e3
|
|
1179
|
+
});
|
|
1180
|
+
if (!json) throw new Error("no output");
|
|
1181
|
+
const repos = JSON.parse(json);
|
|
1182
|
+
const match = repos.find((r) => r.fullName.toLowerCase().endsWith(`/${packageName.toLowerCase()}`) || r.fullName.toLowerCase().endsWith(`/${shortName.toLowerCase()}`));
|
|
1183
|
+
if (match) return `https://github.com/${match.fullName}`;
|
|
1184
|
+
for (const candidate of repos) {
|
|
1185
|
+
const gh = parseGitHubUrl(`https://github.com/${candidate.fullName}`);
|
|
1186
|
+
if (gh && await verifyNpmRepo(gh.owner, gh.repo, packageName)) return `https://github.com/${candidate.fullName}`;
|
|
1187
|
+
}
|
|
1188
|
+
} catch {}
|
|
1189
|
+
const data = await $fetch(`https://api.github.com/search/repositories?q=${encodeURIComponent(`${searchTerm} in:name`)}&per_page=5`).catch(() => null);
|
|
1190
|
+
if (!data?.items?.length) return null;
|
|
1191
|
+
const match = data.items.find((r) => r.full_name.toLowerCase().endsWith(`/${packageName.toLowerCase()}`) || r.full_name.toLowerCase().endsWith(`/${shortName.toLowerCase()}`));
|
|
1192
|
+
if (match) return `https://github.com/${match.full_name}`;
|
|
1193
|
+
for (const candidate of data.items) {
|
|
1194
|
+
const gh = parseGitHubUrl(`https://github.com/${candidate.full_name}`);
|
|
1195
|
+
if (gh && await verifyNpmRepo(gh.owner, gh.repo, packageName)) return `https://github.com/${candidate.full_name}`;
|
|
1196
|
+
}
|
|
1197
|
+
return null;
|
|
1760
1198
|
}
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
const
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
* falls back to GitHub API for private repos.
|
|
1770
|
-
*/
|
|
1771
|
-
async function listFilesAtRef(owner, repo, ref) {
|
|
1199
|
+
async function fetchGitHubRepoMeta(owner, repo, packageName) {
|
|
1200
|
+
const override = packageName ? getDocOverride(packageName) : void 0;
|
|
1201
|
+
if (override?.homepage) return { homepage: override.homepage };
|
|
1202
|
+
const data = await ghApi(`repos/${owner}/${repo}`) ?? await $fetch(`https://api.github.com/repos/${owner}/${repo}`).catch(() => null);
|
|
1203
|
+
return data?.homepage ? { homepage: data.homepage } : null;
|
|
1204
|
+
}
|
|
1205
|
+
async function fetchReadme(owner, repo, subdir, ref) {
|
|
1206
|
+
const branch = ref || "main";
|
|
1772
1207
|
if (!isKnownPrivateRepo(owner, repo)) {
|
|
1773
|
-
const
|
|
1774
|
-
if (
|
|
1208
|
+
const unghUrl = subdir ? `https://ungh.cc/repos/${owner}/${repo}/files/${branch}/${subdir}/README.md` : `https://ungh.cc/repos/${owner}/${repo}/readme${ref ? `?ref=${ref}` : ""}`;
|
|
1209
|
+
if ((await $fetch.raw(unghUrl).catch(() => null))?.ok) return `ungh://${owner}/${repo}${subdir ? `/${subdir}` : ""}${ref ? `@${ref}` : ""}`;
|
|
1775
1210
|
}
|
|
1776
|
-
const
|
|
1777
|
-
|
|
1211
|
+
const basePath = subdir ? `${subdir}/` : "";
|
|
1212
|
+
const branches = ref ? [ref] : ["main", "master"];
|
|
1213
|
+
const token = isKnownPrivateRepo(owner, repo) ? getGitHubToken() : null;
|
|
1214
|
+
const authHeaders = token ? { Authorization: `token ${token}` } : {};
|
|
1215
|
+
for (const b of branches) for (const filename of [
|
|
1216
|
+
"README.md",
|
|
1217
|
+
"Readme.md",
|
|
1218
|
+
"readme.md"
|
|
1219
|
+
]) {
|
|
1220
|
+
const readmeUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${b}/${basePath}${filename}`;
|
|
1221
|
+
if ((await $fetch.raw(readmeUrl, { headers: authHeaders }).catch(() => null))?.ok) return readmeUrl;
|
|
1222
|
+
}
|
|
1223
|
+
const refParam = ref ? `?ref=${ref}` : "";
|
|
1224
|
+
const apiData = await ghApi(subdir ? `repos/${owner}/${repo}/contents/${subdir}/README.md${refParam}` : `repos/${owner}/${repo}/readme${refParam}`);
|
|
1225
|
+
if (apiData?.download_url) {
|
|
1778
1226
|
markRepoPrivate(owner, repo);
|
|
1779
|
-
return
|
|
1227
|
+
return apiData.download_url;
|
|
1780
1228
|
}
|
|
1781
|
-
return
|
|
1229
|
+
return null;
|
|
1782
1230
|
}
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
const candidates = [`v${version}`, version];
|
|
1789
|
-
if (packageName) candidates.push(`${packageName}@${version}`);
|
|
1790
|
-
for (const tag of candidates) {
|
|
1791
|
-
const files = await listFilesAtRef(owner, repo, tag);
|
|
1792
|
-
if (files.length > 0) return {
|
|
1793
|
-
ref: tag,
|
|
1794
|
-
files
|
|
1795
|
-
};
|
|
1231
|
+
async function fetchReadmeContent(url) {
|
|
1232
|
+
if (url.startsWith("file://")) {
|
|
1233
|
+
const filePath = fileURLToPath(url);
|
|
1234
|
+
if (!existsSync(filePath)) return null;
|
|
1235
|
+
return readFileSync(filePath, "utf-8");
|
|
1796
1236
|
}
|
|
1797
|
-
if (
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1237
|
+
if (url.startsWith("ungh://")) {
|
|
1238
|
+
let path = url.replace("ungh://", "");
|
|
1239
|
+
let ref = "main";
|
|
1240
|
+
const atIdx = path.lastIndexOf("@");
|
|
1241
|
+
if (atIdx !== -1) {
|
|
1242
|
+
ref = path.slice(atIdx + 1);
|
|
1243
|
+
path = path.slice(0, atIdx);
|
|
1244
|
+
}
|
|
1245
|
+
const parts = path.split("/");
|
|
1246
|
+
const owner = parts[0];
|
|
1247
|
+
const repo = parts[1];
|
|
1248
|
+
const subdir = parts.slice(2).join("/");
|
|
1249
|
+
const text = await $fetch(subdir ? `https://ungh.cc/repos/${owner}/${repo}/files/${ref}/${subdir}/README.md` : `https://ungh.cc/repos/${owner}/${repo}/readme?ref=${ref}`, { responseType: "text" }).catch(() => null);
|
|
1250
|
+
if (!text) return null;
|
|
1251
|
+
try {
|
|
1252
|
+
const json = JSON.parse(text);
|
|
1253
|
+
return json.markdown || json.file?.contents || null;
|
|
1254
|
+
} catch {
|
|
1255
|
+
return text;
|
|
1805
1256
|
}
|
|
1806
1257
|
}
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1258
|
+
if (url.includes("raw.githubusercontent.com")) return fetchGitHubRaw(url);
|
|
1259
|
+
return fetchText(url);
|
|
1260
|
+
}
|
|
1261
|
+
async function resolveGitHubRepo(owner, repo, onProgress) {
|
|
1262
|
+
onProgress?.("Fetching repo metadata");
|
|
1263
|
+
const repoUrl = `https://github.com/${owner}/${repo}`;
|
|
1264
|
+
const meta = await ghApi(`repos/${owner}/${repo}`) ?? await $fetch(`https://api.github.com/repos/${owner}/${repo}`).catch(() => null);
|
|
1265
|
+
const homepage = meta?.homepage || void 0;
|
|
1266
|
+
const description = meta?.description || void 0;
|
|
1267
|
+
onProgress?.("Fetching latest release");
|
|
1268
|
+
const releases = await fetchUnghReleases(owner, repo);
|
|
1269
|
+
let version = "main";
|
|
1270
|
+
let releasedAt;
|
|
1271
|
+
const latestRelease = releases[0];
|
|
1272
|
+
if (latestRelease) {
|
|
1273
|
+
version = latestRelease.tag.replace(/^v/, "");
|
|
1274
|
+
releasedAt = latestRelease.publishedAt;
|
|
1275
|
+
}
|
|
1276
|
+
onProgress?.("Resolving docs");
|
|
1277
|
+
const gitDocs = await fetchGitDocs(owner, repo, version);
|
|
1278
|
+
const gitDocsUrl = gitDocs ? `${repoUrl}/tree/${gitDocs.ref}/docs` : void 0;
|
|
1279
|
+
const gitRef = gitDocs?.ref;
|
|
1280
|
+
onProgress?.("Fetching README");
|
|
1281
|
+
const readmeUrl = await fetchReadme(owner, repo);
|
|
1282
|
+
let llmsUrl;
|
|
1283
|
+
if (homepage) {
|
|
1284
|
+
onProgress?.("Checking llms.txt");
|
|
1285
|
+
llmsUrl = await fetchLlmsUrl(homepage).catch(() => null) ?? void 0;
|
|
1286
|
+
}
|
|
1287
|
+
if (!gitDocsUrl && !readmeUrl && !llmsUrl) return null;
|
|
1288
|
+
return {
|
|
1289
|
+
name: repo,
|
|
1290
|
+
version: latestRelease ? version : void 0,
|
|
1291
|
+
releasedAt,
|
|
1292
|
+
description,
|
|
1293
|
+
repoUrl,
|
|
1294
|
+
docsUrl: homepage,
|
|
1295
|
+
gitDocsUrl,
|
|
1296
|
+
gitRef,
|
|
1297
|
+
gitDocsFallback: gitDocs?.fallback,
|
|
1298
|
+
readmeUrl: readmeUrl ?? void 0,
|
|
1299
|
+
llmsUrl
|
|
1300
|
+
};
|
|
1301
|
+
}
|
|
1302
|
+
const VALID_CRATE_NAME = /^[a-z0-9][\w-]*$/;
|
|
1303
|
+
const runCratesApiRateLimited = createRateLimitedRunner(1e3);
|
|
1304
|
+
function selectCrateVersion(data, requestedVersion) {
|
|
1305
|
+
const versions = data.versions || [];
|
|
1306
|
+
if (requestedVersion) {
|
|
1307
|
+
const exact = versions.find((v) => v.num === requestedVersion && !v.yanked);
|
|
1308
|
+
if (exact?.num) return {
|
|
1309
|
+
version: exact.num,
|
|
1310
|
+
entry: exact
|
|
1814
1311
|
};
|
|
1815
1312
|
}
|
|
1313
|
+
const crate = data.crate;
|
|
1314
|
+
const preferred = [
|
|
1315
|
+
crate?.max_stable_version,
|
|
1316
|
+
crate?.newest_version,
|
|
1317
|
+
crate?.max_version,
|
|
1318
|
+
crate?.default_version
|
|
1319
|
+
].find(Boolean);
|
|
1320
|
+
if (preferred) {
|
|
1321
|
+
const match = versions.find((v) => v.num === preferred && !v.yanked);
|
|
1322
|
+
if (match?.num) return {
|
|
1323
|
+
version: preferred,
|
|
1324
|
+
entry: match
|
|
1325
|
+
};
|
|
1326
|
+
if (versions.length === 0) return { version: preferred };
|
|
1327
|
+
}
|
|
1328
|
+
const firstStable = versions.find((v) => !v.yanked && v.num);
|
|
1329
|
+
if (firstStable?.num) return {
|
|
1330
|
+
version: firstStable.num,
|
|
1331
|
+
entry: firstStable
|
|
1332
|
+
};
|
|
1816
1333
|
return null;
|
|
1817
1334
|
}
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
async function
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1335
|
+
function pickPreferredUrl(...urls) {
|
|
1336
|
+
return urls.map((v) => v?.trim()).find((v) => !!v);
|
|
1337
|
+
}
|
|
1338
|
+
async function fetchCratesApi(url) {
|
|
1339
|
+
return runCratesApiRateLimited(() => $fetch(url).catch(() => null));
|
|
1340
|
+
}
|
|
1341
|
+
async function resolveCrateDocsWithAttempts(crateName, options = {}) {
|
|
1342
|
+
const attempts = [];
|
|
1343
|
+
const onProgress = options.onProgress;
|
|
1344
|
+
const normalizedName = crateName.trim().toLowerCase();
|
|
1345
|
+
if (!normalizedName || !VALID_CRATE_NAME.test(normalizedName)) {
|
|
1346
|
+
attempts.push({
|
|
1347
|
+
source: "crates",
|
|
1348
|
+
status: "error",
|
|
1349
|
+
message: `Invalid crate name: ${crateName}`
|
|
1350
|
+
});
|
|
1351
|
+
return {
|
|
1352
|
+
package: null,
|
|
1353
|
+
attempts
|
|
1354
|
+
};
|
|
1825
1355
|
}
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1356
|
+
onProgress?.("crates.io metadata");
|
|
1357
|
+
const apiUrl = `https://crates.io/api/v1/crates/${encodeURIComponent(normalizedName)}`;
|
|
1358
|
+
const data = await fetchCratesApi(apiUrl);
|
|
1359
|
+
if (!data?.crate) {
|
|
1360
|
+
attempts.push({
|
|
1361
|
+
source: "crates",
|
|
1362
|
+
url: apiUrl,
|
|
1363
|
+
status: "not-found",
|
|
1364
|
+
message: "Crate not found on crates.io"
|
|
1365
|
+
});
|
|
1366
|
+
return {
|
|
1367
|
+
package: null,
|
|
1368
|
+
attempts
|
|
1369
|
+
};
|
|
1833
1370
|
}
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
const
|
|
1841
|
-
|
|
1371
|
+
attempts.push({
|
|
1372
|
+
source: "crates",
|
|
1373
|
+
url: apiUrl,
|
|
1374
|
+
status: "success",
|
|
1375
|
+
message: `Found crate: ${data.crate.name || normalizedName}`
|
|
1376
|
+
});
|
|
1377
|
+
const selected = selectCrateVersion(data, options.version);
|
|
1378
|
+
if (!selected) {
|
|
1379
|
+
attempts.push({
|
|
1380
|
+
source: "crates",
|
|
1381
|
+
url: apiUrl,
|
|
1382
|
+
status: "error",
|
|
1383
|
+
message: "No usable crate versions found"
|
|
1384
|
+
});
|
|
1385
|
+
return {
|
|
1386
|
+
package: null,
|
|
1387
|
+
attempts
|
|
1388
|
+
};
|
|
1389
|
+
}
|
|
1390
|
+
const version = selected.version;
|
|
1391
|
+
const versionEntry = selected.entry;
|
|
1392
|
+
const docsRsUrl = `https://docs.rs/${encodeURIComponent(normalizedName)}/${encodeURIComponent(version)}`;
|
|
1393
|
+
const repositoryRaw = pickPreferredUrl(versionEntry?.repository, data.crate.repository);
|
|
1394
|
+
const homepage = pickPreferredUrl(versionEntry?.homepage, data.crate.homepage);
|
|
1395
|
+
const documentation = pickPreferredUrl(versionEntry?.documentation, data.crate.documentation);
|
|
1396
|
+
const normalizedRepo = repositoryRaw ? normalizeRepoUrl(repositoryRaw) : void 0;
|
|
1397
|
+
const repoUrl = normalizedRepo && isLikelyCodeHostUrl(normalizedRepo) ? normalizedRepo : isLikelyCodeHostUrl(homepage) ? homepage : void 0;
|
|
1398
|
+
let resolved = {
|
|
1399
|
+
name: normalizedName,
|
|
1400
|
+
version,
|
|
1401
|
+
releasedAt: versionEntry?.created_at || data.crate.updated_at || void 0,
|
|
1402
|
+
description: versionEntry?.description || data.crate.description,
|
|
1403
|
+
docsUrl: (() => {
|
|
1404
|
+
if (documentation && !isUselessDocsUrl(documentation) && !isLikelyCodeHostUrl(documentation)) return documentation;
|
|
1405
|
+
if (homepage && !isUselessDocsUrl(homepage) && !isLikelyCodeHostUrl(homepage)) return homepage;
|
|
1406
|
+
return docsRsUrl;
|
|
1407
|
+
})(),
|
|
1408
|
+
repoUrl
|
|
1409
|
+
};
|
|
1410
|
+
const gh = repoUrl ? parseGitHubUrl(repoUrl) : null;
|
|
1411
|
+
if (gh) {
|
|
1412
|
+
onProgress?.("GitHub enrichment");
|
|
1413
|
+
const ghResolved = await resolveGitHubRepo(gh.owner, gh.repo);
|
|
1414
|
+
if (ghResolved) {
|
|
1415
|
+
attempts.push({
|
|
1416
|
+
source: "github-meta",
|
|
1417
|
+
url: repoUrl,
|
|
1418
|
+
status: "success",
|
|
1419
|
+
message: "Enriched via GitHub repo metadata"
|
|
1420
|
+
});
|
|
1421
|
+
resolved = {
|
|
1422
|
+
...ghResolved,
|
|
1423
|
+
name: normalizedName,
|
|
1424
|
+
version,
|
|
1425
|
+
releasedAt: resolved.releasedAt || ghResolved.releasedAt,
|
|
1426
|
+
description: resolved.description || ghResolved.description,
|
|
1427
|
+
docsUrl: resolved.docsUrl || ghResolved.docsUrl,
|
|
1428
|
+
repoUrl,
|
|
1429
|
+
readmeUrl: ghResolved.readmeUrl || resolved.readmeUrl
|
|
1430
|
+
};
|
|
1431
|
+
} else attempts.push({
|
|
1432
|
+
source: "github-meta",
|
|
1433
|
+
url: repoUrl,
|
|
1434
|
+
status: "not-found",
|
|
1435
|
+
message: "GitHub enrichment failed, using crates.io metadata"
|
|
1436
|
+
});
|
|
1437
|
+
}
|
|
1438
|
+
if (!resolved.llmsUrl && resolved.docsUrl) {
|
|
1439
|
+
onProgress?.("llms.txt discovery");
|
|
1440
|
+
resolved.llmsUrl = await fetchLlmsUrl(resolved.docsUrl).catch(() => null) ?? void 0;
|
|
1441
|
+
if (resolved.llmsUrl) attempts.push({
|
|
1442
|
+
source: "llms.txt",
|
|
1443
|
+
url: resolved.llmsUrl,
|
|
1444
|
+
status: "success"
|
|
1445
|
+
});
|
|
1446
|
+
}
|
|
1447
|
+
return {
|
|
1448
|
+
package: resolved,
|
|
1449
|
+
attempts
|
|
1450
|
+
};
|
|
1842
1451
|
}
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1452
|
+
async function fetchCrawledDocs(url, onProgress, maxPages = 200) {
|
|
1453
|
+
const outputDir = join(tmpdir(), "skilld-crawl", Date.now().toString());
|
|
1454
|
+
onProgress?.(`Crawling ${url}`);
|
|
1455
|
+
const userLang = getUserLang();
|
|
1456
|
+
const foreignUrls = /* @__PURE__ */ new Set();
|
|
1457
|
+
const doCrawl = () => crawlAndGenerate({
|
|
1458
|
+
urls: [url],
|
|
1459
|
+
outputDir,
|
|
1460
|
+
driver: "http",
|
|
1461
|
+
generateLlmsTxt: false,
|
|
1462
|
+
generateIndividualMd: true,
|
|
1463
|
+
maxRequestsPerCrawl: maxPages,
|
|
1464
|
+
onPage: (page) => {
|
|
1465
|
+
const lang = extractHtmlLang(page.html);
|
|
1466
|
+
if (lang && !lang.startsWith("en") && !lang.startsWith(userLang)) foreignUrls.add(page.url);
|
|
1467
|
+
}
|
|
1468
|
+
}, (progress) => {
|
|
1469
|
+
if (progress.crawling.status === "processing" && progress.crawling.total > 0) onProgress?.(`Crawling ${progress.crawling.processed}/${progress.crawling.total} pages`);
|
|
1470
|
+
});
|
|
1471
|
+
let results = await doCrawl().catch((err) => {
|
|
1472
|
+
onProgress?.(`Crawl failed: ${err?.message || err}`);
|
|
1473
|
+
return [];
|
|
1474
|
+
});
|
|
1475
|
+
if (results.length === 0) {
|
|
1476
|
+
onProgress?.("Retrying crawl");
|
|
1477
|
+
results = await doCrawl().catch(() => []);
|
|
1478
|
+
}
|
|
1479
|
+
rmSync(outputDir, {
|
|
1480
|
+
recursive: true,
|
|
1481
|
+
force: true
|
|
1482
|
+
});
|
|
1483
|
+
const docs = [];
|
|
1484
|
+
let localeFiltered = 0;
|
|
1485
|
+
for (const result of results) {
|
|
1486
|
+
if (!result.success || !result.content) continue;
|
|
1487
|
+
if (foreignUrls.has(result.url)) {
|
|
1488
|
+
localeFiltered++;
|
|
1489
|
+
continue;
|
|
1490
|
+
}
|
|
1491
|
+
const segments = (new URL(result.url).pathname.replace(/\/$/, "") || "/index").split("/").filter(Boolean);
|
|
1492
|
+
if (isForeignPathPrefix(segments[0], userLang)) {
|
|
1493
|
+
localeFiltered++;
|
|
1494
|
+
continue;
|
|
1495
|
+
}
|
|
1496
|
+
const path = `docs/${segments.join("/")}.md`;
|
|
1497
|
+
docs.push({
|
|
1498
|
+
path,
|
|
1499
|
+
content: result.content
|
|
1500
|
+
});
|
|
1501
|
+
}
|
|
1502
|
+
if (localeFiltered > 0) onProgress?.(`Filtered ${localeFiltered} foreign locale pages`);
|
|
1503
|
+
onProgress?.(`Crawled ${docs.length} pages`);
|
|
1504
|
+
return docs;
|
|
1848
1505
|
}
|
|
1849
|
-
const
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
"solid",
|
|
1853
|
-
"angular",
|
|
1854
|
-
"svelte",
|
|
1855
|
-
"preact",
|
|
1856
|
-
"lit",
|
|
1857
|
-
"qwik"
|
|
1858
|
-
]);
|
|
1859
|
-
/**
|
|
1860
|
-
* Filter out docs for other frameworks when the package targets a specific one.
|
|
1861
|
-
* e.g. @tanstack/vue-query → keep vue + shared docs, exclude react/solid/angular
|
|
1862
|
-
* Uses word-boundary matching to catch all path conventions:
|
|
1863
|
-
* framework/react/, 0.react/, api/ai-react.md, react-native.mdx, etc.
|
|
1864
|
-
*/
|
|
1865
|
-
function filterFrameworkDocs(files, packageName) {
|
|
1866
|
-
if (!packageName) return files;
|
|
1867
|
-
const shortName = packageName.replace(/^@.*\//, "");
|
|
1868
|
-
const targetFramework = [...FRAMEWORK_NAMES].find((fw) => shortName.includes(fw));
|
|
1869
|
-
if (!targetFramework) return files;
|
|
1870
|
-
const otherFrameworks = [...FRAMEWORK_NAMES].filter((fw) => fw !== targetFramework);
|
|
1871
|
-
const excludePattern = new RegExp(`\\b(?:${otherFrameworks.join("|")})\\b`);
|
|
1872
|
-
return files.filter((f) => !excludePattern.test(f));
|
|
1506
|
+
const HTML_LANG_RE = /<html[^>]*\slang=["']([^"']+)["']/i;
|
|
1507
|
+
function extractHtmlLang(html) {
|
|
1508
|
+
return HTML_LANG_RE.exec(html)?.[1]?.toLowerCase();
|
|
1873
1509
|
}
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
"
|
|
1884
|
-
"
|
|
1885
|
-
"
|
|
1886
|
-
"
|
|
1887
|
-
"
|
|
1888
|
-
"
|
|
1889
|
-
"
|
|
1890
|
-
"
|
|
1891
|
-
"
|
|
1892
|
-
"
|
|
1893
|
-
"
|
|
1894
|
-
"
|
|
1895
|
-
"e2e",
|
|
1896
|
-
"spec",
|
|
1897
|
-
"mocks",
|
|
1898
|
-
"__mocks__"
|
|
1899
|
-
]);
|
|
1900
|
-
/** Directory names that suggest documentation */
|
|
1901
|
-
const DOC_DIR_BONUS = new Set([
|
|
1902
|
-
"docs",
|
|
1903
|
-
"documentation",
|
|
1904
|
-
"pages",
|
|
1905
|
-
"content",
|
|
1906
|
-
"website",
|
|
1907
|
-
"guide",
|
|
1908
|
-
"guides",
|
|
1909
|
-
"wiki",
|
|
1910
|
-
"manual",
|
|
1911
|
-
"api"
|
|
1510
|
+
const LOCALE_CODES = new Set([
|
|
1511
|
+
"ar",
|
|
1512
|
+
"de",
|
|
1513
|
+
"es",
|
|
1514
|
+
"fr",
|
|
1515
|
+
"id",
|
|
1516
|
+
"it",
|
|
1517
|
+
"ja",
|
|
1518
|
+
"ko",
|
|
1519
|
+
"nl",
|
|
1520
|
+
"pl",
|
|
1521
|
+
"pt",
|
|
1522
|
+
"pt-br",
|
|
1523
|
+
"ru",
|
|
1524
|
+
"th",
|
|
1525
|
+
"tr",
|
|
1526
|
+
"uk",
|
|
1527
|
+
"vi",
|
|
1528
|
+
"zh",
|
|
1529
|
+
"zh-cn",
|
|
1530
|
+
"zh-tw"
|
|
1912
1531
|
]);
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
return
|
|
1532
|
+
function isForeignPathPrefix(segment, userLang) {
|
|
1533
|
+
if (!segment) return false;
|
|
1534
|
+
const lower = segment.toLowerCase();
|
|
1535
|
+
if (lower === "en" || lower.startsWith(userLang)) return false;
|
|
1536
|
+
return LOCALE_CODES.has(lower);
|
|
1918
1537
|
}
|
|
1919
|
-
|
|
1920
|
-
|
|
1921
|
-
|
|
1922
|
-
function getPathDepth(path) {
|
|
1923
|
-
return path.split("/").filter(Boolean).length;
|
|
1538
|
+
function getUserLang() {
|
|
1539
|
+
const code = (process.env.LC_ALL || process.env.LANG || process.env.LANGUAGE || "").split(/[_.:-]/)[0]?.toLowerCase() || "";
|
|
1540
|
+
return code.length >= 2 ? code.slice(0, 2) : "en";
|
|
1924
1541
|
}
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
*/
|
|
1928
|
-
function hasDocDirBonus(path) {
|
|
1929
|
-
return path.split("/").some((p) => DOC_DIR_BONUS.has(p.toLowerCase()));
|
|
1542
|
+
function toCrawlPattern(docsUrl) {
|
|
1543
|
+
return `${docsUrl.replace(/\/+$/, "")}/**`;
|
|
1930
1544
|
}
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1545
|
+
const HIGH_VALUE_CATEGORIES = new Set([
|
|
1546
|
+
"q&a",
|
|
1547
|
+
"help",
|
|
1548
|
+
"troubleshooting",
|
|
1549
|
+
"support"
|
|
1550
|
+
]);
|
|
1551
|
+
const LOW_VALUE_CATEGORIES = new Set([
|
|
1552
|
+
"show and tell",
|
|
1553
|
+
"ideas",
|
|
1554
|
+
"polls"
|
|
1555
|
+
]);
|
|
1556
|
+
const TITLE_NOISE_RE = /looking .*(?:developer|engineer|freelanc)|hiring|job post|guide me to (?:complete|finish|build)|help me (?:complete|finish|build)|seeking .* tutorial|recommend.* course/i;
|
|
1557
|
+
const MIN_DISCUSSION_SCORE = 3;
|
|
1558
|
+
function scoreComment(c) {
|
|
1559
|
+
return (c.isMaintainer ? 3 : 1) * (hasCodeBlock(c.body) ? 2 : 1) * (1 + c.reactions);
|
|
1938
1560
|
}
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
prefix: subPkgPrefix
|
|
1953
|
-
};
|
|
1561
|
+
function scoreDiscussion(d) {
|
|
1562
|
+
if (TITLE_NOISE_RE.test(d.title)) return -1;
|
|
1563
|
+
let score = 0;
|
|
1564
|
+
if (d.isMaintainer) score += 3;
|
|
1565
|
+
if (hasCodeBlock([
|
|
1566
|
+
d.body,
|
|
1567
|
+
d.answer || "",
|
|
1568
|
+
...d.topComments.map((c) => c.body)
|
|
1569
|
+
].join("\n"))) score += 3;
|
|
1570
|
+
score += Math.min(d.upvoteCount, 5);
|
|
1571
|
+
if (d.answer) {
|
|
1572
|
+
score += 2;
|
|
1573
|
+
if (d.answer.length > 100) score += 1;
|
|
1954
1574
|
}
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1575
|
+
if (d.topComments.some((c) => c.isMaintainer)) score += 2;
|
|
1576
|
+
if (d.topComments.some((c) => c.reactions > 0)) score += 1;
|
|
1577
|
+
return score;
|
|
1578
|
+
}
|
|
1579
|
+
async function fetchGitHubDiscussions(owner, repo, limit = 20, releasedAt, fromDate) {
|
|
1580
|
+
if (!isGhAvailable()) return [];
|
|
1581
|
+
if (!fromDate && releasedAt) {
|
|
1582
|
+
const cutoff = new Date(releasedAt);
|
|
1583
|
+
cutoff.setMonth(cutoff.getMonth() + 6);
|
|
1584
|
+
if (cutoff < /* @__PURE__ */ new Date()) return [];
|
|
1960
1585
|
}
|
|
1961
|
-
|
|
1962
|
-
const
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1586
|
+
try {
|
|
1587
|
+
const { stdout: result } = spawnSync("gh", [
|
|
1588
|
+
"api",
|
|
1589
|
+
"graphql",
|
|
1590
|
+
"-f",
|
|
1591
|
+
`query=${`query($owner: String!, $repo: String!) { repository(owner: $owner, name: $repo) { discussions(first: ${Math.min(limit * 3, 80)}, orderBy: {field: CREATED_AT, direction: DESC}) { nodes { number title body category { name } createdAt url upvoteCount comments(first: 10) { totalCount nodes { body author { login } authorAssociation reactions { totalCount } } } answer { body author { login } authorAssociation } author { login } authorAssociation } } } }`}`,
|
|
1592
|
+
"-f",
|
|
1593
|
+
`owner=${owner}`,
|
|
1594
|
+
"-f",
|
|
1595
|
+
`repo=${repo}`
|
|
1596
|
+
], {
|
|
1597
|
+
encoding: "utf-8",
|
|
1598
|
+
maxBuffer: 10 * 1024 * 1024
|
|
1599
|
+
});
|
|
1600
|
+
if (!result) return [];
|
|
1601
|
+
const nodes = JSON.parse(result)?.data?.repository?.discussions?.nodes;
|
|
1602
|
+
if (!Array.isArray(nodes)) return [];
|
|
1603
|
+
const fromTs = fromDate ? new Date(fromDate).getTime() : null;
|
|
1604
|
+
return nodes.filter((d) => d.author && !BOT_USERS.has(d.author.login)).filter((d) => {
|
|
1605
|
+
const cat = (d.category?.name || "").toLowerCase();
|
|
1606
|
+
return !LOW_VALUE_CATEGORIES.has(cat);
|
|
1607
|
+
}).filter((d) => !fromTs || new Date(d.createdAt).getTime() >= fromTs).map((d) => {
|
|
1608
|
+
let answer;
|
|
1609
|
+
if (d.answer?.body) {
|
|
1610
|
+
const isMaintainer = [
|
|
1611
|
+
"OWNER",
|
|
1612
|
+
"MEMBER",
|
|
1613
|
+
"COLLABORATOR"
|
|
1614
|
+
].includes(d.answer.authorAssociation);
|
|
1615
|
+
const author = d.answer.author?.login;
|
|
1616
|
+
answer = `${isMaintainer && author ? `**@${author}** [maintainer]:\n\n` : ""}${d.answer.body}`;
|
|
1617
|
+
}
|
|
1618
|
+
const comments = (d.comments?.nodes || []).filter((c) => c.author && !BOT_USERS.has(c.author.login)).filter((c) => !COMMENT_NOISE_RE.test((c.body || "").trim())).map((c) => {
|
|
1619
|
+
const isMaintainer = [
|
|
1620
|
+
"OWNER",
|
|
1621
|
+
"MEMBER",
|
|
1622
|
+
"COLLABORATOR"
|
|
1623
|
+
].includes(c.authorAssociation);
|
|
1624
|
+
return {
|
|
1625
|
+
body: c.body || "",
|
|
1626
|
+
author: c.author.login,
|
|
1627
|
+
reactions: c.reactions?.totalCount || 0,
|
|
1628
|
+
isMaintainer
|
|
1629
|
+
};
|
|
1630
|
+
}).sort((a, b) => scoreComment(b) - scoreComment(a)).slice(0, 3);
|
|
1967
1631
|
return {
|
|
1968
|
-
|
|
1969
|
-
|
|
1632
|
+
number: d.number,
|
|
1633
|
+
title: d.title,
|
|
1634
|
+
body: d.body || "",
|
|
1635
|
+
category: d.category?.name || "",
|
|
1636
|
+
createdAt: d.createdAt,
|
|
1637
|
+
url: d.url,
|
|
1638
|
+
upvoteCount: d.upvoteCount || 0,
|
|
1639
|
+
comments: d.comments?.totalCount || 0,
|
|
1640
|
+
isMaintainer: [
|
|
1641
|
+
"OWNER",
|
|
1642
|
+
"MEMBER",
|
|
1643
|
+
"COLLABORATOR"
|
|
1644
|
+
].includes(d.authorAssociation),
|
|
1645
|
+
answer,
|
|
1646
|
+
topComments: comments
|
|
1970
1647
|
};
|
|
1971
|
-
}
|
|
1972
|
-
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1648
|
+
}).map((d) => ({
|
|
1649
|
+
d,
|
|
1650
|
+
score: scoreDiscussion(d)
|
|
1651
|
+
})).filter(({ score }) => score >= MIN_DISCUSSION_SCORE).sort((a, b) => {
|
|
1652
|
+
const aHigh = HIGH_VALUE_CATEGORIES.has(a.d.category.toLowerCase()) ? 1 : 0;
|
|
1653
|
+
const bHigh = HIGH_VALUE_CATEGORIES.has(b.d.category.toLowerCase()) ? 1 : 0;
|
|
1654
|
+
if (aHigh !== bHigh) return bHigh - aHigh;
|
|
1655
|
+
return b.score - a.score;
|
|
1656
|
+
}).slice(0, limit).map(({ d }) => d);
|
|
1657
|
+
} catch {
|
|
1658
|
+
return [];
|
|
1979
1659
|
}
|
|
1980
|
-
if (dirGroups.size === 0) return null;
|
|
1981
|
-
const scored = Array.from(dirGroups.entries(), ([dir, files]) => ({
|
|
1982
|
-
dir,
|
|
1983
|
-
files,
|
|
1984
|
-
score: scoreDocDir(dir, files.length)
|
|
1985
|
-
})).filter((d) => d.files.length >= 5).sort((a, b) => b.score - a.score);
|
|
1986
|
-
if (scored.length === 0) return null;
|
|
1987
|
-
const best = scored[0];
|
|
1988
|
-
return {
|
|
1989
|
-
files: best.files,
|
|
1990
|
-
prefix: best.dir
|
|
1991
|
-
};
|
|
1992
|
-
}
|
|
1993
|
-
/**
|
|
1994
|
-
* List markdown files in a folder at a specific git ref
|
|
1995
|
-
*/
|
|
1996
|
-
async function listDocsAtRef(owner, repo, ref, pathPrefix = "docs/") {
|
|
1997
|
-
return filterDocFiles(await listFilesAtRef(owner, repo, ref), pathPrefix);
|
|
1998
1660
|
}
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
const discovered = discoverDocFiles(tag.files, packageName);
|
|
2025
|
-
if (discovered) {
|
|
2026
|
-
docs = discovered.files;
|
|
2027
|
-
docsPrefix = discovered.prefix || void 0;
|
|
2028
|
-
allFiles = tag.files;
|
|
1661
|
+
function formatDiscussionAsMarkdown(d) {
|
|
1662
|
+
const fm = buildFrontmatter({
|
|
1663
|
+
number: d.number,
|
|
1664
|
+
title: d.title,
|
|
1665
|
+
category: d.category,
|
|
1666
|
+
created: isoDate(d.createdAt),
|
|
1667
|
+
url: d.url,
|
|
1668
|
+
upvotes: d.upvoteCount,
|
|
1669
|
+
comments: d.comments,
|
|
1670
|
+
answered: !!d.answer
|
|
1671
|
+
});
|
|
1672
|
+
const bodyLimit = d.upvoteCount >= 5 ? 1500 : 800;
|
|
1673
|
+
const lines = [
|
|
1674
|
+
fm,
|
|
1675
|
+
"",
|
|
1676
|
+
`# ${d.title}`
|
|
1677
|
+
];
|
|
1678
|
+
if (d.body) lines.push("", truncateBody(d.body, bodyLimit));
|
|
1679
|
+
if (d.answer) lines.push("", "---", "", "## Accepted Answer", "", truncateBody(d.answer, 1e3));
|
|
1680
|
+
else if (d.topComments.length > 0) {
|
|
1681
|
+
lines.push("", "---", "", "## Top Comments");
|
|
1682
|
+
for (const c of d.topComments) {
|
|
1683
|
+
const reactions = c.reactions > 0 ? ` (+${c.reactions})` : "";
|
|
1684
|
+
const maintainer = c.isMaintainer ? " [maintainer]" : "";
|
|
1685
|
+
lines.push("", `**@${c.author}**${maintainer}${reactions}:`, "", truncateBody(c.body, 600));
|
|
2029
1686
|
}
|
|
2030
1687
|
}
|
|
2031
|
-
|
|
2032
|
-
if (docs.length === 0) return null;
|
|
2033
|
-
return {
|
|
2034
|
-
baseUrl: `https://raw.githubusercontent.com/${owner}/${repo}/${tag.ref}`,
|
|
2035
|
-
ref: tag.ref,
|
|
2036
|
-
files: docs,
|
|
2037
|
-
docsPrefix,
|
|
2038
|
-
allFiles,
|
|
2039
|
-
fallback: tag.fallback
|
|
2040
|
-
};
|
|
2041
|
-
}
|
|
2042
|
-
/**
|
|
2043
|
-
* Strip file extension (.md, .mdx) and leading slash from a path
|
|
2044
|
-
*/
|
|
2045
|
-
function normalizePath(p) {
|
|
2046
|
-
return p.replace(/^\//, "").replace(/\.(?:md|mdx)$/, "");
|
|
1688
|
+
return lines.join("\n");
|
|
2047
1689
|
}
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
return normalizePath(path);
|
|
1690
|
+
function generateDiscussionIndex(discussions) {
|
|
1691
|
+
const byCategory = /* @__PURE__ */ new Map();
|
|
1692
|
+
for (const d of discussions) mapInsert(byCategory, d.category || "Uncategorized", () => []).push(d);
|
|
1693
|
+
const answered = discussions.filter((d) => d.answer).length;
|
|
1694
|
+
const sections = [
|
|
1695
|
+
[
|
|
1696
|
+
"---",
|
|
1697
|
+
`total: ${discussions.length}`,
|
|
1698
|
+
`answered: ${answered}`,
|
|
1699
|
+
"---"
|
|
1700
|
+
].join("\n"),
|
|
1701
|
+
"",
|
|
1702
|
+
"# Discussions Index",
|
|
1703
|
+
""
|
|
1704
|
+
];
|
|
1705
|
+
const cats = [...byCategory.keys()].sort((a, b) => {
|
|
1706
|
+
return (HIGH_VALUE_CATEGORIES.has(a.toLowerCase()) ? 0 : 1) - (HIGH_VALUE_CATEGORIES.has(b.toLowerCase()) ? 0 : 1) || a.localeCompare(b);
|
|
2066
1707
|
});
|
|
2067
|
-
const
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
1708
|
+
for (const cat of cats) {
|
|
1709
|
+
const group = byCategory.get(cat);
|
|
1710
|
+
sections.push(`## ${cat} (${group.length})`, "");
|
|
1711
|
+
for (const d of group) {
|
|
1712
|
+
const upvotes = d.upvoteCount > 0 ? ` (+${d.upvoteCount})` : "";
|
|
1713
|
+
const answered = d.answer ? " [answered]" : "";
|
|
1714
|
+
const date = isoDate(d.createdAt);
|
|
1715
|
+
sections.push(`- [#${d.number}](./discussion-${d.number}.md): ${d.title}${upvotes}${answered} (${date})`);
|
|
1716
|
+
}
|
|
1717
|
+
sections.push("");
|
|
2072
1718
|
}
|
|
2073
|
-
|
|
2074
|
-
return {
|
|
2075
|
-
isValid: matchRatio >= .3,
|
|
2076
|
-
matchRatio
|
|
2077
|
-
};
|
|
1719
|
+
return sections.join("\n");
|
|
2078
1720
|
}
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
const
|
|
2085
|
-
|
|
2086
|
-
"
|
|
2087
|
-
|
|
2088
|
-
|
|
1721
|
+
function generateDocsIndex(docs) {
|
|
1722
|
+
const docFiles = docs.filter((d) => d.path.startsWith("docs/") && d.path.endsWith(".md") && !d.path.endsWith("_INDEX.md")).sort((a, b) => a.path.localeCompare(b.path));
|
|
1723
|
+
if (docFiles.length === 0) return "";
|
|
1724
|
+
const rootFiles = [];
|
|
1725
|
+
const byDir = /* @__PURE__ */ new Map();
|
|
1726
|
+
for (const doc of docFiles) {
|
|
1727
|
+
const rel = doc.path.slice(5);
|
|
1728
|
+
const dir = rel.includes("/") ? rel.slice(0, rel.lastIndexOf("/")) : "";
|
|
1729
|
+
if (!dir) rootFiles.push(doc);
|
|
1730
|
+
else {
|
|
1731
|
+
const list = byDir.get(dir);
|
|
1732
|
+
if (list) list.push(doc);
|
|
1733
|
+
else byDir.set(dir, [doc]);
|
|
1734
|
+
}
|
|
1735
|
+
}
|
|
1736
|
+
const sections = [
|
|
1737
|
+
"---",
|
|
1738
|
+
`total: ${docFiles.length}`,
|
|
1739
|
+
"---",
|
|
1740
|
+
"",
|
|
1741
|
+
"# Docs Index",
|
|
1742
|
+
""
|
|
2089
1743
|
];
|
|
2090
|
-
for (const
|
|
2091
|
-
const
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
1744
|
+
for (const file of rootFiles) {
|
|
1745
|
+
const rel = file.path.slice(5);
|
|
1746
|
+
const title = extractTitle(file.content) || rel.replace(/\.md$/, "");
|
|
1747
|
+
const desc = extractDescription(file.content);
|
|
1748
|
+
const descPart = desc ? `: ${desc}` : "";
|
|
1749
|
+
sections.push(`- [${title}](./${rel})${descPart}`);
|
|
2096
1750
|
}
|
|
2097
|
-
|
|
1751
|
+
if (rootFiles.length > 0) sections.push("");
|
|
1752
|
+
for (const [dir, files] of byDir) {
|
|
1753
|
+
sections.push(`## ${dir} (${files.length})`, "");
|
|
1754
|
+
for (const file of files) {
|
|
1755
|
+
const rel = file.path.slice(5);
|
|
1756
|
+
const title = extractTitle(file.content) || rel.replace(/\.md$/, "").split("/").pop();
|
|
1757
|
+
const desc = extractDescription(file.content);
|
|
1758
|
+
const descPart = desc ? `: ${desc}` : "";
|
|
1759
|
+
sections.push(`- [${title}](./${rel})${descPart}`);
|
|
1760
|
+
}
|
|
1761
|
+
sections.push("");
|
|
1762
|
+
}
|
|
1763
|
+
return sections.join("\n");
|
|
2098
1764
|
}
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
1765
|
+
const SKIP_DIRS = [
|
|
1766
|
+
"node_modules",
|
|
1767
|
+
"_vendor",
|
|
1768
|
+
"__tests__",
|
|
1769
|
+
"__mocks__",
|
|
1770
|
+
"__fixtures__",
|
|
1771
|
+
"test",
|
|
1772
|
+
"tests",
|
|
1773
|
+
"fixture",
|
|
1774
|
+
"fixtures",
|
|
1775
|
+
"locales",
|
|
1776
|
+
"locale",
|
|
1777
|
+
"i18n",
|
|
1778
|
+
".git"
|
|
1779
|
+
];
|
|
1780
|
+
const SKIP_PATTERNS = [
|
|
1781
|
+
"*.min.*",
|
|
1782
|
+
"*.prod.*",
|
|
1783
|
+
"*.global.*",
|
|
1784
|
+
"*.browser.*",
|
|
1785
|
+
"*.map",
|
|
1786
|
+
"*.map.js",
|
|
1787
|
+
"CHANGELOG*",
|
|
1788
|
+
"LICENSE*",
|
|
1789
|
+
"README*"
|
|
1790
|
+
];
|
|
1791
|
+
const MAX_FILE_SIZE = 500 * 1024;
|
|
1792
|
+
async function resolveEntryFiles(packageDir) {
|
|
1793
|
+
if (!existsSync(join(packageDir, "package.json"))) return [];
|
|
1794
|
+
const files = await glob(["**/*.d.{ts,mts,cts}"], {
|
|
1795
|
+
cwd: packageDir,
|
|
1796
|
+
ignore: [...SKIP_DIRS.map((d) => `**/${d}/**`), ...SKIP_PATTERNS],
|
|
1797
|
+
absolute: false,
|
|
1798
|
+
expandDirectories: false
|
|
1799
|
+
});
|
|
1800
|
+
const entries = [];
|
|
1801
|
+
for (const file of files) {
|
|
1802
|
+
const absPath = join(packageDir, file);
|
|
1803
|
+
let content;
|
|
1804
|
+
try {
|
|
1805
|
+
content = readFileSync(absPath, "utf-8");
|
|
1806
|
+
} catch {
|
|
2104
1807
|
continue;
|
|
2105
1808
|
}
|
|
2106
|
-
if (
|
|
2107
|
-
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
"search",
|
|
2112
|
-
"repos",
|
|
2113
|
-
searchTerm,
|
|
2114
|
-
"--json",
|
|
2115
|
-
"fullName",
|
|
2116
|
-
"--limit",
|
|
2117
|
-
"5"
|
|
2118
|
-
], {
|
|
2119
|
-
encoding: "utf-8",
|
|
2120
|
-
timeout: 15e3
|
|
1809
|
+
if (content.length > MAX_FILE_SIZE) continue;
|
|
1810
|
+
entries.push({
|
|
1811
|
+
path: file,
|
|
1812
|
+
content,
|
|
1813
|
+
type: "types"
|
|
2121
1814
|
});
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
1815
|
+
}
|
|
1816
|
+
return entries;
|
|
1817
|
+
}
|
|
1818
|
+
function parseGitSkillInput(input) {
|
|
1819
|
+
const trimmed = input.trim();
|
|
1820
|
+
if (trimmed.startsWith("@")) return null;
|
|
1821
|
+
if (trimmed.startsWith("./") || trimmed.startsWith("../") || trimmed.startsWith("/") || trimmed.startsWith("~")) return {
|
|
1822
|
+
type: "local",
|
|
1823
|
+
localPath: trimmed.startsWith("~") ? resolve(process.env.HOME || "", trimmed.slice(1)) : resolve(trimmed)
|
|
1824
|
+
};
|
|
1825
|
+
if (trimmed.startsWith("git@")) {
|
|
1826
|
+
const gh = parseGitHubUrl(normalizeRepoUrl(trimmed));
|
|
1827
|
+
if (gh) return {
|
|
1828
|
+
type: "github",
|
|
1829
|
+
owner: gh.owner,
|
|
1830
|
+
repo: gh.repo
|
|
1831
|
+
};
|
|
1832
|
+
return null;
|
|
1833
|
+
}
|
|
1834
|
+
if (trimmed.startsWith("https://") || trimmed.startsWith("http://")) return parseGitUrl(trimmed);
|
|
1835
|
+
if (/^[\w.-]+\/[\w.-]+$/.test(trimmed)) return {
|
|
1836
|
+
type: "github",
|
|
1837
|
+
owner: trimmed.split("/")[0],
|
|
1838
|
+
repo: trimmed.split("/")[1]
|
|
1839
|
+
};
|
|
1840
|
+
return null;
|
|
1841
|
+
}
|
|
1842
|
+
function parseGitUrl(url) {
|
|
1843
|
+
try {
|
|
1844
|
+
const parsed = new URL(url);
|
|
1845
|
+
if (parsed.hostname === "github.com" || parsed.hostname === "www.github.com") {
|
|
1846
|
+
const parts = parsed.pathname.replace(/^\//, "").replace(/\.git$/, "").split("/");
|
|
1847
|
+
const owner = parts[0];
|
|
1848
|
+
const repo = parts[1];
|
|
1849
|
+
if (!owner || !repo) return null;
|
|
1850
|
+
if (parts[2] === "tree" && parts.length >= 4) return {
|
|
1851
|
+
type: "github",
|
|
1852
|
+
owner,
|
|
1853
|
+
repo,
|
|
1854
|
+
ref: parts[3],
|
|
1855
|
+
skillPath: parts.length > 4 ? parts.slice(4).join("/") : void 0
|
|
1856
|
+
};
|
|
1857
|
+
return {
|
|
1858
|
+
type: "github",
|
|
1859
|
+
owner,
|
|
1860
|
+
repo
|
|
1861
|
+
};
|
|
2129
1862
|
}
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
1863
|
+
if (parsed.hostname === "gitlab.com") {
|
|
1864
|
+
const parts = parsed.pathname.replace(/^\//, "").replace(/\.git$/, "").split("/");
|
|
1865
|
+
const owner = parts[0];
|
|
1866
|
+
const repo = parts[1];
|
|
1867
|
+
if (!owner || !repo) return null;
|
|
1868
|
+
return {
|
|
1869
|
+
type: "gitlab",
|
|
1870
|
+
owner,
|
|
1871
|
+
repo
|
|
1872
|
+
};
|
|
1873
|
+
}
|
|
1874
|
+
return null;
|
|
1875
|
+
} catch {
|
|
1876
|
+
return null;
|
|
2138
1877
|
}
|
|
2139
|
-
return null;
|
|
2140
1878
|
}
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
if (override?.homepage) return { homepage: override.homepage };
|
|
2148
|
-
const data = await ghApi(`repos/${owner}/${repo}`) ?? await $fetch(`https://api.github.com/repos/${owner}/${repo}`).catch(() => null);
|
|
2149
|
-
return data?.homepage ? { homepage: data.homepage } : null;
|
|
1879
|
+
function parseSkillFrontmatterName(content) {
|
|
1880
|
+
const fm = parseFrontmatter(content);
|
|
1881
|
+
return {
|
|
1882
|
+
name: fm.name,
|
|
1883
|
+
description: fm.description
|
|
1884
|
+
};
|
|
2150
1885
|
}
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
if (
|
|
1886
|
+
function collectFiles(dir, prefix = "") {
|
|
1887
|
+
const files = [];
|
|
1888
|
+
if (!existsSync(dir)) return files;
|
|
1889
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
1890
|
+
const relPath = prefix ? `${prefix}/${entry.name}` : entry.name;
|
|
1891
|
+
const fullPath = resolve(dir, entry.name);
|
|
1892
|
+
if (entry.isDirectory()) files.push(...collectFiles(fullPath, relPath));
|
|
1893
|
+
else if (entry.isFile()) files.push({
|
|
1894
|
+
path: relPath,
|
|
1895
|
+
content: readFileSync(fullPath, "utf-8")
|
|
1896
|
+
});
|
|
2159
1897
|
}
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
1898
|
+
return files;
|
|
1899
|
+
}
|
|
1900
|
+
async function fetchGitSkills(source, onProgress) {
|
|
1901
|
+
if (source.type === "local") return fetchLocalSkills(source);
|
|
1902
|
+
if (source.type === "github") return fetchGitHubSkills(source, onProgress);
|
|
1903
|
+
if (source.type === "gitlab") return fetchGitLabSkills(source, onProgress);
|
|
1904
|
+
return { skills: [] };
|
|
1905
|
+
}
|
|
1906
|
+
function fetchLocalSkills(source) {
|
|
1907
|
+
const base = source.localPath;
|
|
1908
|
+
if (!existsSync(base)) return { skills: [] };
|
|
1909
|
+
const skills = [];
|
|
1910
|
+
const skillsDir = resolve(base, "skills");
|
|
1911
|
+
if (existsSync(skillsDir)) for (const entry of readdirSync(skillsDir, { withFileTypes: true })) {
|
|
1912
|
+
if (!entry.isDirectory()) continue;
|
|
1913
|
+
const skill = readLocalSkill(resolve(skillsDir, entry.name), `skills/${entry.name}`);
|
|
1914
|
+
if (skill) skills.push(skill);
|
|
2171
1915
|
}
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
markRepoPrivate(owner, repo);
|
|
2176
|
-
return apiData.download_url;
|
|
1916
|
+
if (skills.length === 0) {
|
|
1917
|
+
const skill = readLocalSkill(base, "");
|
|
1918
|
+
if (skill) skills.push(skill);
|
|
2177
1919
|
}
|
|
2178
|
-
return
|
|
1920
|
+
return { skills };
|
|
2179
1921
|
}
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
1922
|
+
function readLocalSkill(dir, repoPath) {
|
|
1923
|
+
const skillMdPath = resolve(dir, "SKILL.md");
|
|
1924
|
+
if (!existsSync(skillMdPath)) return null;
|
|
1925
|
+
const content = readFileSync(skillMdPath, "utf-8");
|
|
1926
|
+
const frontmatter = parseSkillFrontmatterName(content);
|
|
1927
|
+
const dirName = dir.split("/").pop();
|
|
1928
|
+
const name = frontmatter.name || dirName;
|
|
1929
|
+
const files = collectFiles(dir).filter((f) => f.path !== "SKILL.md");
|
|
1930
|
+
return {
|
|
1931
|
+
name,
|
|
1932
|
+
description: frontmatter.description || "",
|
|
1933
|
+
path: repoPath,
|
|
1934
|
+
content,
|
|
1935
|
+
files
|
|
1936
|
+
};
|
|
1937
|
+
}
|
|
1938
|
+
async function fetchGitHubSkills(source, onProgress) {
|
|
1939
|
+
const { owner, repo } = source;
|
|
1940
|
+
if (!owner || !repo) return { skills: [] };
|
|
1941
|
+
const ref = source.ref || "main";
|
|
1942
|
+
const refs = ref === "main" ? ["main", "master"] : [ref];
|
|
1943
|
+
for (const tryRef of refs) {
|
|
1944
|
+
const skills = await downloadGitHubSkills(owner, repo, tryRef, source.skillPath, onProgress);
|
|
1945
|
+
if (skills.length > 0) return { skills };
|
|
2188
1946
|
}
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
1947
|
+
return { skills: [] };
|
|
1948
|
+
}
|
|
1949
|
+
async function downloadGitHubSkills(owner, repo, ref, skillPath, onProgress) {
|
|
1950
|
+
const tempDir = join(tmpdir(), `skilld-${Date.now()}`);
|
|
1951
|
+
try {
|
|
1952
|
+
if (skillPath) {
|
|
1953
|
+
onProgress?.(`Downloading ${owner}/${repo}/${skillPath}@${ref}`);
|
|
1954
|
+
const { dir } = await downloadTemplate(`github:${owner}/${repo}/${skillPath}#${ref}`, {
|
|
1955
|
+
dir: tempDir,
|
|
1956
|
+
force: true,
|
|
1957
|
+
auth: getGitHubToken() || void 0
|
|
1958
|
+
});
|
|
1959
|
+
const skill = readLocalSkill(dir, skillPath);
|
|
1960
|
+
return skill ? [skill] : [];
|
|
2196
1961
|
}
|
|
2197
|
-
|
|
2198
|
-
const owner = parts[0];
|
|
2199
|
-
const repo = parts[1];
|
|
2200
|
-
const subdir = parts.slice(2).join("/");
|
|
2201
|
-
const text = await $fetch(subdir ? `https://ungh.cc/repos/${owner}/${repo}/files/${ref}/${subdir}/README.md` : `https://ungh.cc/repos/${owner}/${repo}/readme?ref=${ref}`, { responseType: "text" }).catch(() => null);
|
|
2202
|
-
if (!text) return null;
|
|
1962
|
+
onProgress?.(`Downloading ${owner}/${repo}/skills@${ref}`);
|
|
2203
1963
|
try {
|
|
2204
|
-
const
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
1964
|
+
const { dir } = await downloadTemplate(`github:${owner}/${repo}/skills#${ref}`, {
|
|
1965
|
+
dir: tempDir,
|
|
1966
|
+
force: true,
|
|
1967
|
+
auth: getGitHubToken() || void 0
|
|
1968
|
+
});
|
|
1969
|
+
const skills = [];
|
|
1970
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
1971
|
+
if (!entry.isDirectory()) continue;
|
|
1972
|
+
const skill = readLocalSkill(resolve(dir, entry.name), `skills/${entry.name}`);
|
|
1973
|
+
if (skill) skills.push(skill);
|
|
1974
|
+
}
|
|
1975
|
+
if (skills.length > 0) {
|
|
1976
|
+
onProgress?.(`Found ${skills.length} skill(s)`);
|
|
1977
|
+
return skills;
|
|
1978
|
+
}
|
|
1979
|
+
} catch {}
|
|
1980
|
+
const content = await fetchGitHubRaw(`https://raw.githubusercontent.com/${owner}/${repo}/${ref}/SKILL.md`);
|
|
1981
|
+
if (content) {
|
|
1982
|
+
const fm = parseSkillFrontmatterName(content);
|
|
1983
|
+
onProgress?.("Found 1 skill");
|
|
1984
|
+
return [{
|
|
1985
|
+
name: fm.name || repo,
|
|
1986
|
+
description: fm.description || "",
|
|
1987
|
+
path: "",
|
|
1988
|
+
content,
|
|
1989
|
+
files: []
|
|
1990
|
+
}];
|
|
2208
1991
|
}
|
|
1992
|
+
return [];
|
|
1993
|
+
} catch {
|
|
1994
|
+
return [];
|
|
1995
|
+
} finally {
|
|
1996
|
+
rmSync(tempDir, {
|
|
1997
|
+
recursive: true,
|
|
1998
|
+
force: true
|
|
1999
|
+
});
|
|
2209
2000
|
}
|
|
2210
|
-
if (url.includes("raw.githubusercontent.com")) return fetchGitHubRaw(url);
|
|
2211
|
-
return fetchText(url);
|
|
2212
2001
|
}
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2002
|
+
async function fetchGitLabSkills(source, onProgress) {
|
|
2003
|
+
const { owner, repo } = source;
|
|
2004
|
+
if (!owner || !repo) return { skills: [] };
|
|
2005
|
+
const ref = source.ref || "main";
|
|
2006
|
+
const tempDir = join(tmpdir(), `skilld-gitlab-${Date.now()}`);
|
|
2007
|
+
try {
|
|
2008
|
+
const subdir = source.skillPath || "skills";
|
|
2009
|
+
onProgress?.(`Downloading ${owner}/${repo}/${subdir}@${ref}`);
|
|
2010
|
+
const { dir } = await downloadTemplate(`gitlab:${owner}/${repo}/${subdir}#${ref}`, {
|
|
2011
|
+
dir: tempDir,
|
|
2012
|
+
force: true
|
|
2013
|
+
});
|
|
2014
|
+
if (source.skillPath) {
|
|
2015
|
+
const skill = readLocalSkill(dir, source.skillPath);
|
|
2016
|
+
return { skills: skill ? [skill] : [] };
|
|
2017
|
+
}
|
|
2018
|
+
const skills = [];
|
|
2019
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
2020
|
+
if (!entry.isDirectory()) continue;
|
|
2021
|
+
const skill = readLocalSkill(resolve(dir, entry.name), `skills/${entry.name}`);
|
|
2022
|
+
if (skill) skills.push(skill);
|
|
2023
|
+
}
|
|
2024
|
+
if (skills.length > 0) {
|
|
2025
|
+
onProgress?.(`Found ${skills.length} skill(s)`);
|
|
2026
|
+
return { skills };
|
|
2027
|
+
}
|
|
2028
|
+
const content = await $fetch(`https://gitlab.com/${owner}/${repo}/-/raw/${ref}/SKILL.md`, { responseType: "text" }).catch(() => null);
|
|
2029
|
+
if (content) {
|
|
2030
|
+
const fm = parseSkillFrontmatterName(content);
|
|
2031
|
+
return { skills: [{
|
|
2032
|
+
name: fm.name || repo,
|
|
2033
|
+
description: fm.description || "",
|
|
2034
|
+
path: "",
|
|
2035
|
+
content,
|
|
2036
|
+
files: []
|
|
2037
|
+
}] };
|
|
2038
|
+
}
|
|
2039
|
+
return { skills: [] };
|
|
2040
|
+
} catch {
|
|
2041
|
+
return { skills: [] };
|
|
2042
|
+
} finally {
|
|
2043
|
+
rmSync(tempDir, {
|
|
2044
|
+
recursive: true,
|
|
2045
|
+
force: true
|
|
2046
|
+
});
|
|
2242
2047
|
}
|
|
2243
|
-
if (!gitDocsUrl && !readmeUrl && !llmsUrl) return null;
|
|
2244
|
-
return {
|
|
2245
|
-
name: repo,
|
|
2246
|
-
version: latestRelease ? version : void 0,
|
|
2247
|
-
releasedAt,
|
|
2248
|
-
description,
|
|
2249
|
-
repoUrl,
|
|
2250
|
-
docsUrl: homepage,
|
|
2251
|
-
gitDocsUrl,
|
|
2252
|
-
gitRef,
|
|
2253
|
-
gitDocsFallback: gitDocs?.fallback,
|
|
2254
|
-
readmeUrl: readmeUrl ?? void 0,
|
|
2255
|
-
llmsUrl
|
|
2256
|
-
};
|
|
2257
2048
|
}
|
|
2258
|
-
//#endregion
|
|
2259
|
-
//#region src/sources/npm.ts
|
|
2260
|
-
/**
|
|
2261
|
-
* Search npm registry for packages matching a query.
|
|
2262
|
-
* Used as a fallback when direct package lookup fails.
|
|
2263
|
-
*/
|
|
2264
2049
|
async function searchNpmPackages(query, size = 5) {
|
|
2265
2050
|
const data = await $fetch(`https://registry.npmjs.org/-/v1/search?text=${encodeURIComponent(query)}&size=${size}`).catch(() => null);
|
|
2266
2051
|
if (!data?.objects?.length) return [];
|
|
@@ -2270,17 +2055,11 @@ async function searchNpmPackages(query, size = 5) {
|
|
|
2270
2055
|
version: o.package.version
|
|
2271
2056
|
}));
|
|
2272
2057
|
}
|
|
2273
|
-
/**
|
|
2274
|
-
* Fetch package info from npm registry
|
|
2275
|
-
*/
|
|
2276
2058
|
async function fetchNpmPackage(packageName) {
|
|
2277
2059
|
const data = await $fetch(`https://unpkg.com/${packageName}/package.json`).catch(() => null);
|
|
2278
2060
|
if (data) return data;
|
|
2279
2061
|
return $fetch(`https://registry.npmjs.org/${packageName}/latest`).catch(() => null);
|
|
2280
2062
|
}
|
|
2281
|
-
/**
|
|
2282
|
-
* Fetch release date and dist-tags from npm registry
|
|
2283
|
-
*/
|
|
2284
2063
|
async function fetchNpmRegistryMeta(packageName, version) {
|
|
2285
2064
|
const { name: barePackageName } = parsePackageSpec(packageName);
|
|
2286
2065
|
const data = await $fetch(`https://registry.npmjs.org/${barePackageName}`, { headers: { Accept: "application/vnd.npm.install-v1+json" } }).catch(() => null);
|
|
@@ -2294,10 +2073,6 @@ async function fetchNpmRegistryMeta(packageName, version) {
|
|
|
2294
2073
|
distTags
|
|
2295
2074
|
};
|
|
2296
2075
|
}
|
|
2297
|
-
/**
|
|
2298
|
-
* Shared GitHub resolution cascade: git docs → repo meta (homepage) → README.
|
|
2299
|
-
* Used for both "repo URL found in package.json" and "repo URL found via search" paths.
|
|
2300
|
-
*/
|
|
2301
2076
|
async function resolveGitHub(gh, targetVersion, pkg, result, attempts, onProgress, opts) {
|
|
2302
2077
|
let allFiles;
|
|
2303
2078
|
if (targetVersion) {
|
|
@@ -2356,15 +2131,9 @@ async function resolveGitHub(gh, targetVersion, pkg, result, attempts, onProgres
|
|
|
2356
2131
|
});
|
|
2357
2132
|
return allFiles;
|
|
2358
2133
|
}
|
|
2359
|
-
/**
|
|
2360
|
-
* Resolve documentation URL for a package (legacy - returns null on failure)
|
|
2361
|
-
*/
|
|
2362
2134
|
async function resolvePackageDocs(packageName, options = {}) {
|
|
2363
2135
|
return (await resolvePackageDocsWithAttempts(packageName, options)).package;
|
|
2364
2136
|
}
|
|
2365
|
-
/**
|
|
2366
|
-
* Resolve documentation URL for a package with attempt tracking
|
|
2367
|
-
*/
|
|
2368
2137
|
async function resolvePackageDocsWithAttempts(packageName, options = {}) {
|
|
2369
2138
|
const attempts = [];
|
|
2370
2139
|
const { onProgress } = options;
|
|
@@ -2500,9 +2269,6 @@ async function resolvePackageDocsWithAttempts(packageName, options = {}) {
|
|
|
2500
2269
|
registryVersion: pkg.version
|
|
2501
2270
|
};
|
|
2502
2271
|
}
|
|
2503
|
-
/**
|
|
2504
|
-
* Parse version specifier, handling protocols like link:, workspace:, npm:, file:
|
|
2505
|
-
*/
|
|
2506
2272
|
function parseVersionSpecifier(name, version, cwd) {
|
|
2507
2273
|
if (version.startsWith("link:")) {
|
|
2508
2274
|
const linkedPkg = readPackageJsonSafe(join(resolve(cwd, version.slice(5)), "package.json"));
|
|
@@ -2537,10 +2303,6 @@ function parseVersionSpecifier(name, version, cwd) {
|
|
|
2537
2303
|
};
|
|
2538
2304
|
return null;
|
|
2539
2305
|
}
|
|
2540
|
-
/**
|
|
2541
|
-
* Resolve the actual installed version of a package by finding its package.json
|
|
2542
|
-
* via mlly's resolvePathSync. Works regardless of package manager or version protocol.
|
|
2543
|
-
*/
|
|
2544
2306
|
function resolveInstalledVersion(name, cwd) {
|
|
2545
2307
|
try {
|
|
2546
2308
|
return readPackageJsonSafe(resolvePathSync(`${name}/package.json`, { url: cwd }))?.parsed.version || null;
|
|
@@ -2558,9 +2320,6 @@ function resolveInstalledVersion(name, cwd) {
|
|
|
2558
2320
|
return null;
|
|
2559
2321
|
}
|
|
2560
2322
|
}
|
|
2561
|
-
/**
|
|
2562
|
-
* Read package.json dependencies with versions
|
|
2563
|
-
*/
|
|
2564
2323
|
async function readLocalDependencies(cwd) {
|
|
2565
2324
|
const result = readPackageJsonSafe(join(cwd, "package.json"));
|
|
2566
2325
|
if (!result) throw new Error("No package.json found in current directory");
|
|
@@ -2576,9 +2335,6 @@ async function readLocalDependencies(cwd) {
|
|
|
2576
2335
|
}
|
|
2577
2336
|
return results;
|
|
2578
2337
|
}
|
|
2579
|
-
/**
|
|
2580
|
-
* Read package info from a local path (for link: deps)
|
|
2581
|
-
*/
|
|
2582
2338
|
function readLocalPackageInfo(localPath) {
|
|
2583
2339
|
const result = readPackageJsonSafe(join(localPath, "package.json"));
|
|
2584
2340
|
if (!result) return null;
|
|
@@ -2594,9 +2350,6 @@ function readLocalPackageInfo(localPath) {
|
|
|
2594
2350
|
localPath
|
|
2595
2351
|
};
|
|
2596
2352
|
}
|
|
2597
|
-
/**
|
|
2598
|
-
* Resolve docs for a local package (link: dependency)
|
|
2599
|
-
*/
|
|
2600
2353
|
async function resolveLocalPackageDocs(localPath) {
|
|
2601
2354
|
const info = readLocalPackageInfo(localPath);
|
|
2602
2355
|
if (!info) return null;
|
|
@@ -2626,13 +2379,6 @@ async function resolveLocalPackageDocs(localPath) {
|
|
|
2626
2379
|
if (!result.readmeUrl && !result.gitDocsUrl) return null;
|
|
2627
2380
|
return result;
|
|
2628
2381
|
}
|
|
2629
|
-
/**
|
|
2630
|
-
* Download and extract npm package tarball to cache directory.
|
|
2631
|
-
* Used when the package isn't available in node_modules.
|
|
2632
|
-
*
|
|
2633
|
-
* Extracts to: ~/.skilld/references/<pkg>@<version>/pkg/
|
|
2634
|
-
* Returns the extracted directory path, or null on failure.
|
|
2635
|
-
*/
|
|
2636
2382
|
async function fetchPkgDist(name, version) {
|
|
2637
2383
|
const cacheDir = getCacheDir(name, version);
|
|
2638
2384
|
const pkgDir = join(cacheDir, "pkg");
|
|
@@ -2641,7 +2387,7 @@ async function fetchPkgDist(name, version) {
|
|
|
2641
2387
|
if (!data) return null;
|
|
2642
2388
|
const tarballUrl = data.dist?.tarball;
|
|
2643
2389
|
if (!tarballUrl) return null;
|
|
2644
|
-
const tarballRes = await fetch(tarballUrl, { headers: { "User-Agent":
|
|
2390
|
+
const tarballRes = await fetch(tarballUrl, { headers: { "User-Agent": SKILLD_USER_AGENT } }).catch(() => null);
|
|
2645
2391
|
if (!tarballRes?.ok || !tarballRes.body) return null;
|
|
2646
2392
|
mkdirSync(pkgDir, { recursive: true });
|
|
2647
2393
|
const tmpTarball = join(cacheDir, "_pkg.tgz");
|
|
@@ -2700,23 +2446,16 @@ async function fetchPkgDist(name, version) {
|
|
|
2700
2446
|
} catch {}
|
|
2701
2447
|
}
|
|
2702
2448
|
}
|
|
2703
|
-
/**
|
|
2704
|
-
* Fetch just the latest version string from npm (lightweight)
|
|
2705
|
-
*/
|
|
2706
2449
|
async function fetchLatestVersion(packageName) {
|
|
2707
2450
|
const data = await $fetch(`https://unpkg.com/${packageName}/package.json`).catch(() => null);
|
|
2708
2451
|
if (data?.version) return data.version;
|
|
2709
2452
|
return (await $fetch(`https://registry.npmjs.org/${packageName}`, { headers: { Accept: "application/vnd.npm.install-v1+json" } }).catch(() => null))?.["dist-tags"]?.latest || null;
|
|
2710
2453
|
}
|
|
2711
|
-
/**
|
|
2712
|
-
* Get installed skill version from SKILL.md
|
|
2713
|
-
*/
|
|
2714
2454
|
function getInstalledSkillVersion(skillDir) {
|
|
2715
2455
|
const skillPath = join(skillDir, "SKILL.md");
|
|
2716
2456
|
if (!existsSync(skillPath)) return null;
|
|
2717
2457
|
return readFileSync(skillPath, "utf-8").match(/^version:\s*"?([^"\n]+)"?/m)?.[1] || null;
|
|
2718
2458
|
}
|
|
2719
|
-
|
|
2720
|
-
export { isGitHubRepoUrl as $, parseGitSkillInput as A, isGhAvailable as B, downloadLlmsDocs as C, normalizeLlmsLinks as D, fetchLlmsUrl as E, formatDiscussionAsMarkdown as F, fetchReleaseNotes as G, toCrawlPattern as H, generateDiscussionIndex as I, parseSemver as J, generateReleaseIndex as K, fetchGitHubIssues as L, resolveEntryFiles as M, generateDocsIndex as N, parseMarkdownLinks as O, fetchGitHubDiscussions as P, fetchText as Q, formatIssueAsMarkdown as R, validateGitDocsWithLlms as S, fetchLlmsTxt as T, fetchBlogReleases as U, fetchCrawledDocs as V, compareSemver as W, extractBranchHint as X, $fetch as Y, fetchGitHubRaw as Z, fetchReadme as _, getInstalledSkillVersion as a, isShallowGitDocs as b, readLocalPackageInfo as c, resolvePackageDocs as d, normalizeRepoUrl as et, resolvePackageDocsWithAttempts as f, fetchGitHubRepoMeta as g, fetchGitDocs as h, fetchPkgDist as i, parseSkillFrontmatterName as j, fetchGitSkills as k, resolveInstalledVersion as l, MIN_GIT_DOCS as m, fetchNpmPackage as n, parsePackageSpec as nt, parseVersionSpecifier as o, searchNpmPackages as p, isPrerelease as q, fetchNpmRegistryMeta as r, verifyUrl as rt, readLocalDependencies as s, fetchLatestVersion as t, parseGitHubUrl as tt, resolveLocalPackageDocs as u, fetchReadmeContent as v, extractSections as w, resolveGitHubRepo as x, filterFrameworkDocs as y, generateIssueIndex as z };
|
|
2459
|
+
export { fetchText as $, filterFrameworkDocs as A, fetchGitHubIssues as B, toCrawlPattern as C, fetchGitHubRepoMeta as D, fetchGitDocs as E, extractSections as F, compareSemver as G, generateIssueIndex as H, fetchLlmsTxt as I, isPrerelease as J, fetchReleaseNotes as K, fetchLlmsUrl as L, resolveGitHubRepo as M, validateGitDocsWithLlms as N, fetchReadme as O, downloadLlmsDocs as P, fetchGitHubRaw as Q, normalizeLlmsLinks as R, fetchCrawledDocs as S, MIN_GIT_DOCS as T, isGhAvailable as U, formatIssueAsMarkdown as V, fetchBlogReleases as W, $fetch as X, parseSemver as Y, extractBranchHint as Z, resolveEntryFiles as _, getInstalledSkillVersion as a, formatDiscussionAsMarkdown as b, readLocalPackageInfo as c, resolvePackageDocs as d, isGitHubRepoUrl as et, resolvePackageDocsWithAttempts as f, parseSkillFrontmatterName as g, parseGitSkillInput as h, fetchPkgDist as i, verifyUrl as it, isShallowGitDocs as j, fetchReadmeContent as k, resolveInstalledVersion as l, fetchGitSkills as m, fetchNpmPackage as n, parseGitHubUrl as nt, parseVersionSpecifier as o, searchNpmPackages as p, generateReleaseIndex as q, fetchNpmRegistryMeta as r, parsePackageSpec as rt, readLocalDependencies as s, fetchLatestVersion as t, normalizeRepoUrl as tt, resolveLocalPackageDocs as u, generateDocsIndex as v, resolveCrateDocsWithAttempts as w, generateDiscussionIndex as x, fetchGitHubDiscussions as y, parseMarkdownLinks as z };
|
|
2721
2460
|
|
|
2722
2461
|
//# sourceMappingURL=sources.mjs.map
|