skilld 1.1.2 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_chunks/index2.d.mts +11 -2
- package/dist/_chunks/index2.d.mts.map +1 -1
- package/dist/_chunks/install.mjs +2 -2
- package/dist/_chunks/install.mjs.map +1 -1
- package/dist/_chunks/sources.mjs +959 -872
- package/dist/_chunks/sources.mjs.map +1 -1
- package/dist/_chunks/sync.mjs +2 -2
- package/dist/_chunks/sync.mjs.map +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/sources/index.d.mts +2 -2
- package/dist/sources/index.mjs +2 -2
- package/dist/types.d.mts +1 -1
- package/package.json +1 -1
package/dist/_chunks/sources.mjs
CHANGED
|
@@ -34,969 +34,1050 @@ function buildFrontmatter(fields) {
|
|
|
34
34
|
lines.push("---");
|
|
35
35
|
return lines.join("\n");
|
|
36
36
|
}
|
|
37
|
-
|
|
38
|
-
//#region src/sources/issues.ts
|
|
37
|
+
let _ghToken;
|
|
39
38
|
/**
|
|
40
|
-
* GitHub
|
|
41
|
-
*
|
|
42
|
-
* Categorized by labels, noise filtered out, non-technical issues detected
|
|
39
|
+
* Get GitHub auth token from gh CLI (cached).
|
|
40
|
+
* Returns null if gh CLI is not available or not authenticated.
|
|
43
41
|
*/
|
|
44
|
-
|
|
42
|
+
function getGitHubToken() {
|
|
43
|
+
if (_ghToken !== void 0) return _ghToken;
|
|
44
|
+
try {
|
|
45
|
+
const { stdout } = spawnSync("gh", ["auth", "token"], {
|
|
46
|
+
encoding: "utf-8",
|
|
47
|
+
timeout: 5e3,
|
|
48
|
+
stdio: [
|
|
49
|
+
"ignore",
|
|
50
|
+
"pipe",
|
|
51
|
+
"ignore"
|
|
52
|
+
]
|
|
53
|
+
});
|
|
54
|
+
_ghToken = stdout?.trim() || null;
|
|
55
|
+
} catch {
|
|
56
|
+
_ghToken = null;
|
|
57
|
+
}
|
|
58
|
+
return _ghToken;
|
|
59
|
+
}
|
|
60
|
+
/** Repos where ungh.cc failed but gh api succeeded (likely private) */
|
|
61
|
+
const _needsAuth = /* @__PURE__ */ new Set();
|
|
62
|
+
/** Mark a repo as needing authenticated access */
|
|
63
|
+
function markRepoPrivate(owner, repo) {
|
|
64
|
+
_needsAuth.add(`${owner}/${repo}`);
|
|
65
|
+
}
|
|
66
|
+
/** Check if a repo is known to need authenticated access */
|
|
67
|
+
function isKnownPrivateRepo(owner, repo) {
|
|
68
|
+
return _needsAuth.has(`${owner}/${repo}`);
|
|
69
|
+
}
|
|
70
|
+
const GH_API = "https://api.github.com";
|
|
71
|
+
const ghApiFetch = ofetch.create({
|
|
72
|
+
retry: 2,
|
|
73
|
+
retryDelay: 500,
|
|
74
|
+
timeout: 15e3,
|
|
75
|
+
headers: { "User-Agent": "skilld/1.0" }
|
|
76
|
+
});
|
|
77
|
+
const LINK_NEXT_RE = /<([^>]+)>;\s*rel="next"/;
|
|
78
|
+
/** Parse GitHub Link header for next page URL */
|
|
79
|
+
function parseLinkNext(header) {
|
|
80
|
+
if (!header) return null;
|
|
81
|
+
return header.match(LINK_NEXT_RE)?.[1] ?? null;
|
|
82
|
+
}
|
|
45
83
|
/**
|
|
46
|
-
*
|
|
84
|
+
* Authenticated fetch against api.github.com. Returns null if no token or request fails.
|
|
85
|
+
* Endpoint should be relative, e.g. `repos/owner/repo/releases`.
|
|
47
86
|
*/
|
|
48
|
-
function
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
return
|
|
87
|
+
async function ghApi(endpoint) {
|
|
88
|
+
const token = getGitHubToken();
|
|
89
|
+
if (!token) return null;
|
|
90
|
+
return ghApiFetch(`${GH_API}/${endpoint}`, { headers: { Authorization: `token ${token}` } }).catch(() => null);
|
|
52
91
|
}
|
|
53
|
-
/** Labels that indicate noise — filter these out entirely */
|
|
54
|
-
const NOISE_LABELS = new Set([
|
|
55
|
-
"duplicate",
|
|
56
|
-
"stale",
|
|
57
|
-
"invalid",
|
|
58
|
-
"wontfix",
|
|
59
|
-
"won't fix",
|
|
60
|
-
"spam",
|
|
61
|
-
"off-topic",
|
|
62
|
-
"needs triage",
|
|
63
|
-
"triage"
|
|
64
|
-
]);
|
|
65
|
-
/** Labels that indicate feature requests — deprioritize */
|
|
66
|
-
const FEATURE_LABELS = new Set([
|
|
67
|
-
"enhancement",
|
|
68
|
-
"feature",
|
|
69
|
-
"feature request",
|
|
70
|
-
"feature-request",
|
|
71
|
-
"proposal",
|
|
72
|
-
"rfc",
|
|
73
|
-
"idea",
|
|
74
|
-
"suggestion"
|
|
75
|
-
]);
|
|
76
|
-
const BUG_LABELS = new Set([
|
|
77
|
-
"bug",
|
|
78
|
-
"defect",
|
|
79
|
-
"regression",
|
|
80
|
-
"error",
|
|
81
|
-
"crash",
|
|
82
|
-
"fix",
|
|
83
|
-
"confirmed",
|
|
84
|
-
"verified"
|
|
85
|
-
]);
|
|
86
|
-
const QUESTION_LABELS = new Set([
|
|
87
|
-
"question",
|
|
88
|
-
"help wanted",
|
|
89
|
-
"support",
|
|
90
|
-
"usage",
|
|
91
|
-
"how-to",
|
|
92
|
-
"help",
|
|
93
|
-
"assistance"
|
|
94
|
-
]);
|
|
95
|
-
const DOCS_LABELS = new Set([
|
|
96
|
-
"documentation",
|
|
97
|
-
"docs",
|
|
98
|
-
"doc",
|
|
99
|
-
"typo"
|
|
100
|
-
]);
|
|
101
92
|
/**
|
|
102
|
-
*
|
|
103
|
-
*
|
|
93
|
+
* Paginated GitHub API fetch. Follows Link headers, returns concatenated arrays.
|
|
94
|
+
* Endpoint should return a JSON array, e.g. `repos/owner/repo/releases`.
|
|
104
95
|
*/
|
|
105
|
-
function
|
|
106
|
-
|
|
107
|
-
return
|
|
96
|
+
async function ghApiPaginated(endpoint) {
|
|
97
|
+
const token = getGitHubToken();
|
|
98
|
+
if (!token) return [];
|
|
99
|
+
const headers = { Authorization: `token ${token}` };
|
|
100
|
+
const results = [];
|
|
101
|
+
let url = `${GH_API}/${endpoint}`;
|
|
102
|
+
while (url) {
|
|
103
|
+
const res = await ghApiFetch.raw(url, { headers }).catch(() => null);
|
|
104
|
+
if (!res?.ok || !Array.isArray(res._data)) break;
|
|
105
|
+
results.push(...res._data);
|
|
106
|
+
url = parseLinkNext(res.headers.get("link"));
|
|
107
|
+
}
|
|
108
|
+
return results;
|
|
108
109
|
}
|
|
110
|
+
//#endregion
|
|
111
|
+
//#region src/sources/utils.ts
|
|
109
112
|
/**
|
|
110
|
-
*
|
|
113
|
+
* Shared utilities for doc resolution
|
|
111
114
|
*/
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
return "other";
|
|
119
|
-
}
|
|
115
|
+
const $fetch = ofetch.create({
|
|
116
|
+
retry: 3,
|
|
117
|
+
retryDelay: 500,
|
|
118
|
+
timeout: 15e3,
|
|
119
|
+
headers: { "User-Agent": "skilld/1.0" }
|
|
120
|
+
});
|
|
120
121
|
/**
|
|
121
|
-
*
|
|
122
|
+
* Fetch text content from URL
|
|
122
123
|
*/
|
|
123
|
-
function
|
|
124
|
-
|
|
125
|
-
if (issue.title.startsWith("☂️") || issue.title.startsWith("[META]") || issue.title.startsWith("[Tracking]")) return true;
|
|
126
|
-
return false;
|
|
124
|
+
async function fetchText(url) {
|
|
125
|
+
return $fetch(url, { responseType: "text" }).catch(() => null);
|
|
127
126
|
}
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
127
|
+
const RAW_GH_RE = /raw\.githubusercontent\.com\/([^/]+)\/([^/]+)/;
|
|
128
|
+
/** Extract owner/repo from a GitHub raw content URL */
|
|
129
|
+
function extractGitHubRepo(url) {
|
|
130
|
+
const match = url.match(RAW_GH_RE);
|
|
131
|
+
return match ? {
|
|
132
|
+
owner: match[1],
|
|
133
|
+
repo: match[2]
|
|
134
|
+
} : null;
|
|
131
135
|
}
|
|
132
136
|
/**
|
|
133
|
-
*
|
|
134
|
-
*
|
|
135
|
-
*
|
|
137
|
+
* Fetch text from a GitHub raw URL with auth fallback for private repos.
|
|
138
|
+
* Tries unauthenticated first (fast path), falls back to authenticated
|
|
139
|
+
* request when the repo is known to be private or unauthenticated fails.
|
|
140
|
+
*
|
|
141
|
+
* Only sends auth tokens to raw.githubusercontent.com — returns null for
|
|
142
|
+
* non-GitHub URLs that fail unauthenticated to prevent token leakage.
|
|
136
143
|
*/
|
|
137
|
-
function
|
|
138
|
-
const
|
|
139
|
-
if (
|
|
140
|
-
|
|
141
|
-
|
|
144
|
+
async function fetchGitHubRaw(url) {
|
|
145
|
+
const gh = extractGitHubRepo(url);
|
|
146
|
+
if (!(gh ? isKnownPrivateRepo(gh.owner, gh.repo) : false)) {
|
|
147
|
+
const content = await fetchText(url);
|
|
148
|
+
if (content) return content;
|
|
149
|
+
}
|
|
150
|
+
if (!gh) return null;
|
|
151
|
+
const token = getGitHubToken();
|
|
152
|
+
if (!token) return null;
|
|
153
|
+
const content = await $fetch(url, {
|
|
154
|
+
responseType: "text",
|
|
155
|
+
headers: { Authorization: `token ${token}` }
|
|
156
|
+
}).catch(() => null);
|
|
157
|
+
if (content) markRepoPrivate(gh.owner, gh.repo);
|
|
158
|
+
return content;
|
|
142
159
|
}
|
|
143
160
|
/**
|
|
144
|
-
*
|
|
145
|
-
* Steep decay so recent issues dominate over old high-reaction ones.
|
|
146
|
-
* At 0.6: 1yr=0.63x, 2yr=0.45x, 4yr=0.29x, 6yr=0.22x
|
|
161
|
+
* Verify URL exists and is not HTML (likely 404 page)
|
|
147
162
|
*/
|
|
148
|
-
function
|
|
149
|
-
|
|
163
|
+
async function verifyUrl(url) {
|
|
164
|
+
const res = await $fetch.raw(url, { method: "HEAD" }).catch(() => null);
|
|
165
|
+
if (!res) return false;
|
|
166
|
+
return !(res.headers.get("content-type") || "").includes("text/html");
|
|
150
167
|
}
|
|
151
168
|
/**
|
|
152
|
-
*
|
|
153
|
-
* Bugs and questions get priority; feature requests are hard-capped.
|
|
169
|
+
* Check if URL points to a social media or package registry site (not real docs)
|
|
154
170
|
*/
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
for (let i = 0; i < take; i++) {
|
|
173
|
-
selected.push(group[i]);
|
|
174
|
-
used.add(group[i].number);
|
|
175
|
-
remaining--;
|
|
176
|
-
}
|
|
171
|
+
const USELESS_HOSTS = new Set([
|
|
172
|
+
"twitter.com",
|
|
173
|
+
"x.com",
|
|
174
|
+
"facebook.com",
|
|
175
|
+
"linkedin.com",
|
|
176
|
+
"youtube.com",
|
|
177
|
+
"instagram.com",
|
|
178
|
+
"npmjs.com",
|
|
179
|
+
"www.npmjs.com",
|
|
180
|
+
"yarnpkg.com"
|
|
181
|
+
]);
|
|
182
|
+
function isUselessDocsUrl(url) {
|
|
183
|
+
try {
|
|
184
|
+
const { hostname } = new URL(url);
|
|
185
|
+
return USELESS_HOSTS.has(hostname);
|
|
186
|
+
} catch {
|
|
187
|
+
return false;
|
|
177
188
|
}
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Check if URL is a GitHub repo URL (not a docs site)
|
|
192
|
+
*/
|
|
193
|
+
function isGitHubRepoUrl(url) {
|
|
194
|
+
try {
|
|
195
|
+
const parsed = new URL(url);
|
|
196
|
+
return parsed.hostname === "github.com" || parsed.hostname === "www.github.com";
|
|
197
|
+
} catch {
|
|
198
|
+
return false;
|
|
185
199
|
}
|
|
186
|
-
return selected.sort((a, b) => b.score - a.score);
|
|
187
200
|
}
|
|
188
201
|
/**
|
|
189
|
-
*
|
|
202
|
+
* Parse owner/repo from GitHub URL
|
|
190
203
|
*/
|
|
191
|
-
function
|
|
192
|
-
|
|
193
|
-
if (
|
|
194
|
-
return
|
|
204
|
+
function parseGitHubUrl(url) {
|
|
205
|
+
const match = url.match(/github\.com\/([^/]+)\/([^/]+?)(?:\.git)?(?:[/#]|$)/);
|
|
206
|
+
if (!match) return null;
|
|
207
|
+
return {
|
|
208
|
+
owner: match[1],
|
|
209
|
+
repo: match[2]
|
|
210
|
+
};
|
|
195
211
|
}
|
|
196
212
|
/**
|
|
197
|
-
*
|
|
198
|
-
* Instead of slicing at a char limit, finds a safe break point.
|
|
213
|
+
* Normalize git repo URL to https
|
|
199
214
|
*/
|
|
200
|
-
function
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
215
|
+
function normalizeRepoUrl(url) {
|
|
216
|
+
return url.replace(/^git\+/, "").replace(/#.*$/, "").replace(/\.git$/, "").replace(/^git:\/\//, "https://").replace(/^ssh:\/\/git@github\.com/, "https://github.com").replace(/^git@github\.com:/, "https://github.com/");
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Parse package spec with optional dist-tag or version: "vue@beta" → { name: "vue", tag: "beta" }
|
|
220
|
+
* Handles scoped packages: "@vue/reactivity@beta" → { name: "@vue/reactivity", tag: "beta" }
|
|
221
|
+
*/
|
|
222
|
+
function parsePackageSpec(spec) {
|
|
223
|
+
if (spec.startsWith("@")) {
|
|
224
|
+
const slashIdx = spec.indexOf("/");
|
|
225
|
+
if (slashIdx !== -1) {
|
|
226
|
+
const atIdx = spec.indexOf("@", slashIdx + 1);
|
|
227
|
+
if (atIdx !== -1) return {
|
|
228
|
+
name: spec.slice(0, atIdx),
|
|
229
|
+
tag: spec.slice(atIdx + 1)
|
|
230
|
+
};
|
|
212
231
|
}
|
|
232
|
+
return { name: spec };
|
|
213
233
|
}
|
|
214
|
-
const
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
234
|
+
const atIdx = spec.indexOf("@");
|
|
235
|
+
if (atIdx !== -1) return {
|
|
236
|
+
name: spec.slice(0, atIdx),
|
|
237
|
+
tag: spec.slice(atIdx + 1)
|
|
238
|
+
};
|
|
239
|
+
return { name: spec };
|
|
218
240
|
}
|
|
219
241
|
/**
|
|
220
|
-
*
|
|
242
|
+
* Extract branch hint from URL fragment (e.g. "git+https://...#main" → "main")
|
|
221
243
|
*/
|
|
222
|
-
function
|
|
223
|
-
const
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
date.setMonth(date.getMonth() + 6);
|
|
229
|
-
datePart = `+closed:<=${isoDate(date.toISOString())}`;
|
|
230
|
-
} else datePart = `+closed:>${oneYearAgo()}`;
|
|
231
|
-
else if (releasedAt) {
|
|
232
|
-
const date = new Date(releasedAt);
|
|
233
|
-
date.setMonth(date.getMonth() + 6);
|
|
234
|
-
datePart = `+created:<=${isoDate(date.toISOString())}`;
|
|
235
|
-
}
|
|
236
|
-
const { stdout: result } = spawnSync("gh", [
|
|
237
|
-
"api",
|
|
238
|
-
`search/issues?q=${`repo:${owner}/${repo}+is:issue+is:${state}${datePart}`}&sort=reactions&order=desc&per_page=${fetchCount}`,
|
|
239
|
-
"-q",
|
|
240
|
-
".items[] | {number, title, state, labels: [.labels[]?.name], body, createdAt: .created_at, url: .html_url, reactions: .reactions[\"+1\"], comments: .comments, user: .user.login, userType: .user.type, authorAssociation: .author_association}"
|
|
241
|
-
], {
|
|
242
|
-
encoding: "utf-8",
|
|
243
|
-
maxBuffer: 10 * 1024 * 1024
|
|
244
|
-
});
|
|
245
|
-
if (!result) return [];
|
|
246
|
-
return result.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)).filter((issue) => !BOT_USERS.has(issue.user) && issue.userType !== "Bot").filter((issue) => !isNoiseIssue(issue)).filter((issue) => !isNonTechnical(issue)).map(({ user: _, userType: __, authorAssociation, ...issue }) => {
|
|
247
|
-
const isMaintainer = [
|
|
248
|
-
"OWNER",
|
|
249
|
-
"MEMBER",
|
|
250
|
-
"COLLABORATOR"
|
|
251
|
-
].includes(authorAssociation);
|
|
252
|
-
const isRoadmap = /\broadmap\b/i.test(issue.title) || issue.labels.some((l) => /roadmap/i.test(l));
|
|
253
|
-
return {
|
|
254
|
-
...issue,
|
|
255
|
-
type: classifyIssue(issue.labels),
|
|
256
|
-
topComments: [],
|
|
257
|
-
score: freshnessScore(issue.reactions, issue.createdAt) * (isMaintainer && isRoadmap ? 5 : 1)
|
|
258
|
-
};
|
|
259
|
-
}).sort((a, b) => b.score - a.score).slice(0, count);
|
|
260
|
-
}
|
|
261
|
-
function oneYearAgo() {
|
|
262
|
-
const d = /* @__PURE__ */ new Date();
|
|
263
|
-
d.setFullYear(d.getFullYear() - 1);
|
|
264
|
-
return isoDate(d.toISOString());
|
|
244
|
+
function extractBranchHint(url) {
|
|
245
|
+
const hash = url.indexOf("#");
|
|
246
|
+
if (hash === -1) return void 0;
|
|
247
|
+
const fragment = url.slice(hash + 1);
|
|
248
|
+
if (!fragment || fragment === "readme") return void 0;
|
|
249
|
+
return fragment;
|
|
265
250
|
}
|
|
266
|
-
|
|
267
|
-
|
|
251
|
+
//#endregion
|
|
252
|
+
//#region src/sources/releases.ts
|
|
268
253
|
/**
|
|
269
|
-
*
|
|
270
|
-
* Enriches the top N highest-score issues with their best comments.
|
|
271
|
-
* Prioritizes: comments with code blocks, from maintainers, with high reactions.
|
|
272
|
-
* Filters out "+1", "any updates?", "same here" noise.
|
|
254
|
+
* GitHub release notes fetching via GitHub API (preferred) with ungh.cc fallback
|
|
273
255
|
*/
|
|
274
|
-
function
|
|
275
|
-
const
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
"-f",
|
|
285
|
-
`owner=${owner}`,
|
|
286
|
-
"-f",
|
|
287
|
-
`repo=${repo}`
|
|
288
|
-
], {
|
|
289
|
-
encoding: "utf-8",
|
|
290
|
-
maxBuffer: 10 * 1024 * 1024
|
|
291
|
-
});
|
|
292
|
-
if (!result) return;
|
|
293
|
-
const repo_ = JSON.parse(result)?.data?.repository;
|
|
294
|
-
if (!repo_) return;
|
|
295
|
-
for (let i = 0; i < worth.length; i++) {
|
|
296
|
-
const nodes = repo_[`i${i}`]?.comments?.nodes;
|
|
297
|
-
if (!Array.isArray(nodes)) continue;
|
|
298
|
-
const issue = worth[i];
|
|
299
|
-
const comments = nodes.filter((c) => c.author && !BOT_USERS.has(c.author.login)).filter((c) => !COMMENT_NOISE_RE$1.test((c.body || "").trim())).map((c) => {
|
|
300
|
-
const isMaintainer = [
|
|
301
|
-
"OWNER",
|
|
302
|
-
"MEMBER",
|
|
303
|
-
"COLLABORATOR"
|
|
304
|
-
].includes(c.authorAssociation);
|
|
305
|
-
const body = c.body || "";
|
|
306
|
-
const reactions = c.reactions?.totalCount || 0;
|
|
307
|
-
const _score = (isMaintainer ? 3 : 1) * (hasCodeBlock$1(body) ? 2 : 1) * (1 + reactions);
|
|
308
|
-
return {
|
|
309
|
-
body,
|
|
310
|
-
author: c.author.login,
|
|
311
|
-
reactions,
|
|
312
|
-
isMaintainer,
|
|
313
|
-
_score
|
|
314
|
-
};
|
|
315
|
-
}).sort((a, b) => b._score - a._score);
|
|
316
|
-
issue.topComments = comments.slice(0, 3).map(({ _score: _, ...c }) => c);
|
|
317
|
-
if (issue.state === "closed") issue.resolvedIn = detectResolvedVersion(comments);
|
|
318
|
-
}
|
|
319
|
-
} catch {}
|
|
256
|
+
function parseSemver(version) {
|
|
257
|
+
const clean = version.replace(/^v/, "");
|
|
258
|
+
const match = clean.match(/^(\d+)(?:\.(\d+))?(?:\.(\d+))?/);
|
|
259
|
+
if (!match) return null;
|
|
260
|
+
return {
|
|
261
|
+
major: +match[1],
|
|
262
|
+
minor: match[2] ? +match[2] : 0,
|
|
263
|
+
patch: match[3] ? +match[3] : 0,
|
|
264
|
+
raw: clean
|
|
265
|
+
};
|
|
320
266
|
}
|
|
321
267
|
/**
|
|
322
|
-
*
|
|
323
|
-
*
|
|
268
|
+
* Extract version from a release tag, handling monorepo formats:
|
|
269
|
+
* - `pkg@1.2.3` → `1.2.3`
|
|
270
|
+
* - `pkg-v1.2.3` → `1.2.3`
|
|
271
|
+
* - `v1.2.3` → `1.2.3`
|
|
272
|
+
* - `1.2.3` → `1.2.3`
|
|
324
273
|
*/
|
|
325
|
-
function
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
if (
|
|
331
|
-
const vMatch = c.body.match(/\bv?(\d+\.\d+\.\d+)\b/);
|
|
332
|
-
if (vMatch) return vMatch[1];
|
|
333
|
-
}
|
|
274
|
+
function extractVersion(tag, packageName) {
|
|
275
|
+
if (packageName) {
|
|
276
|
+
const atMatch = tag.match(new RegExp(`^${escapeRegex(packageName)}@(.+)$`));
|
|
277
|
+
if (atMatch) return atMatch[1];
|
|
278
|
+
const dashMatch = tag.match(new RegExp(`^${escapeRegex(packageName)}-v?(.+)$`));
|
|
279
|
+
if (dashMatch) return dashMatch[1];
|
|
334
280
|
}
|
|
281
|
+
return tag.replace(/^v/, "");
|
|
282
|
+
}
|
|
283
|
+
function escapeRegex(str) {
|
|
284
|
+
return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
335
285
|
}
|
|
336
286
|
/**
|
|
337
|
-
*
|
|
338
|
-
* Returns a balanced mix: bugs > questions > docs > other > features.
|
|
339
|
-
* Filters noise, non-technical content, and enriches with quality comments.
|
|
287
|
+
* Check if a release tag belongs to a specific package
|
|
340
288
|
*/
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
const openCount = Math.ceil(limit * .75);
|
|
344
|
-
const closedCount = limit - openCount;
|
|
345
|
-
try {
|
|
346
|
-
const open = fetchIssuesByState(owner, repo, "open", Math.min(openCount * 2, 100), releasedAt, fromDate);
|
|
347
|
-
const closed = fetchIssuesByState(owner, repo, "closed", Math.min(closedCount * 2, 50), releasedAt, fromDate);
|
|
348
|
-
const selected = applyTypeQuotas([...open, ...closed], limit);
|
|
349
|
-
enrichWithComments(owner, repo, selected);
|
|
350
|
-
return selected;
|
|
351
|
-
} catch {
|
|
352
|
-
return [];
|
|
353
|
-
}
|
|
289
|
+
function tagMatchesPackage(tag, packageName) {
|
|
290
|
+
return tag.startsWith(`${packageName}@`) || tag.startsWith(`${packageName}-v`) || tag.startsWith(`${packageName}-`);
|
|
354
291
|
}
|
|
355
292
|
/**
|
|
356
|
-
*
|
|
293
|
+
* Check if a version string contains a prerelease suffix (e.g. 6.0.0-beta, 1.2.3-rc.1)
|
|
357
294
|
*/
|
|
358
|
-
function
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
295
|
+
function isPrerelease(version) {
|
|
296
|
+
return /^\d+\.\d+\.\d+-.+/.test(version.replace(/^v/, ""));
|
|
297
|
+
}
|
|
298
|
+
function compareSemver(a, b) {
|
|
299
|
+
if (a.major !== b.major) return a.major - b.major;
|
|
300
|
+
if (a.minor !== b.minor) return a.minor - b.minor;
|
|
301
|
+
return a.patch - b.patch;
|
|
302
|
+
}
|
|
303
|
+
/** Map GitHub API release to our GitHubRelease shape */
|
|
304
|
+
function mapApiRelease(r) {
|
|
305
|
+
return {
|
|
306
|
+
id: r.id,
|
|
307
|
+
tag: r.tag_name,
|
|
308
|
+
name: r.name,
|
|
309
|
+
prerelease: r.prerelease,
|
|
310
|
+
createdAt: r.created_at,
|
|
311
|
+
publishedAt: r.published_at,
|
|
312
|
+
markdown: r.body
|
|
369
313
|
};
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
314
|
+
}
|
|
315
|
+
/**
|
|
316
|
+
* Fetch all releases — GitHub API first (authenticated, async), ungh.cc fallback
|
|
317
|
+
*/
|
|
318
|
+
async function fetchAllReleases(owner, repo) {
|
|
319
|
+
const apiReleases = await ghApiPaginated(`repos/${owner}/${repo}/releases`);
|
|
320
|
+
if (apiReleases.length > 0) return apiReleases.map(mapApiRelease);
|
|
321
|
+
return (await $fetch(`https://ungh.cc/repos/${owner}/${repo}/releases`, { signal: AbortSignal.timeout(15e3) }).catch(() => null))?.releases ?? [];
|
|
322
|
+
}
|
|
323
|
+
/**
|
|
324
|
+
* Select last 20 stable releases for a package, sorted newest first.
|
|
325
|
+
* For monorepos, filters to package-specific tags (pkg@version).
|
|
326
|
+
* Falls back to generic tags (v1.2.3) only if no package-specific found.
|
|
327
|
+
* If installedVersion is provided, filters out releases newer than it.
|
|
328
|
+
*/
|
|
329
|
+
function selectReleases(releases, packageName, installedVersion, fromDate) {
|
|
330
|
+
const hasMonorepoTags = packageName && releases.some((r) => tagMatchesPackage(r.tag, packageName));
|
|
331
|
+
const installedSv = installedVersion ? parseSemver(installedVersion) : null;
|
|
332
|
+
const installedIsPrerelease = installedVersion ? isPrerelease(installedVersion) : false;
|
|
333
|
+
const fromTs = fromDate ? new Date(fromDate).getTime() : null;
|
|
334
|
+
const sorted = releases.filter((r) => {
|
|
335
|
+
const ver = extractVersion(r.tag, hasMonorepoTags ? packageName : void 0);
|
|
336
|
+
if (!ver) return false;
|
|
337
|
+
const sv = parseSemver(ver);
|
|
338
|
+
if (!sv) return false;
|
|
339
|
+
if (hasMonorepoTags && packageName && !tagMatchesPackage(r.tag, packageName)) return false;
|
|
340
|
+
if (fromTs) {
|
|
341
|
+
const pubDate = r.publishedAt || r.createdAt;
|
|
342
|
+
if (pubDate && new Date(pubDate).getTime() < fromTs) return false;
|
|
388
343
|
}
|
|
389
|
-
|
|
390
|
-
|
|
344
|
+
if (r.prerelease) {
|
|
345
|
+
if (!installedIsPrerelease || !installedSv) return false;
|
|
346
|
+
return sv.major === installedSv.major && sv.minor === installedSv.minor;
|
|
347
|
+
}
|
|
348
|
+
if (installedSv && compareSemver(sv, installedSv) > 0) return false;
|
|
349
|
+
return true;
|
|
350
|
+
}).sort((a, b) => {
|
|
351
|
+
const verA = extractVersion(a.tag, hasMonorepoTags ? packageName : void 0);
|
|
352
|
+
const verB = extractVersion(b.tag, hasMonorepoTags ? packageName : void 0);
|
|
353
|
+
if (!verA || !verB) return 0;
|
|
354
|
+
return compareSemver(parseSemver(verB), parseSemver(verA));
|
|
355
|
+
});
|
|
356
|
+
return fromDate ? sorted : sorted.slice(0, 20);
|
|
391
357
|
}
|
|
392
358
|
/**
|
|
393
|
-
*
|
|
394
|
-
* Groups by type so the LLM can quickly find bugs vs questions.
|
|
359
|
+
* Format a release as markdown with YAML frontmatter
|
|
395
360
|
*/
|
|
396
|
-
function
|
|
397
|
-
const
|
|
398
|
-
|
|
399
|
-
const
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
other: "Other"
|
|
405
|
-
};
|
|
406
|
-
const typeOrder = [
|
|
407
|
-
"bug",
|
|
408
|
-
"question",
|
|
409
|
-
"docs",
|
|
410
|
-
"other",
|
|
411
|
-
"feature"
|
|
361
|
+
function formatRelease(release, packageName) {
|
|
362
|
+
const date = isoDate(release.publishedAt || release.createdAt);
|
|
363
|
+
const version = extractVersion(release.tag, packageName) || release.tag;
|
|
364
|
+
const fm = [
|
|
365
|
+
"---",
|
|
366
|
+
`tag: ${release.tag}`,
|
|
367
|
+
`version: ${version}`,
|
|
368
|
+
`published: ${date}`
|
|
412
369
|
];
|
|
413
|
-
|
|
370
|
+
if (release.name && release.name !== release.tag) fm.push(`name: "${release.name.replace(/"/g, "\\\"")}"`);
|
|
371
|
+
fm.push("---");
|
|
372
|
+
return `${fm.join("\n")}\n\n# ${release.name || release.tag}\n\n${release.markdown}`;
|
|
373
|
+
}
|
|
374
|
+
/**
|
|
375
|
+
* Generate a unified summary index of all releases for quick LLM scanning.
|
|
376
|
+
* Includes GitHub releases, blog release posts, and CHANGELOG link.
|
|
377
|
+
*/
|
|
378
|
+
function generateReleaseIndex(releasesOrOpts, packageName) {
|
|
379
|
+
const opts = Array.isArray(releasesOrOpts) ? {
|
|
380
|
+
releases: releasesOrOpts,
|
|
381
|
+
packageName
|
|
382
|
+
} : releasesOrOpts;
|
|
383
|
+
const { releases, blogReleases, hasChangelog } = opts;
|
|
384
|
+
const pkg = opts.packageName;
|
|
385
|
+
const lines = [
|
|
414
386
|
[
|
|
415
387
|
"---",
|
|
416
|
-
`total: ${
|
|
417
|
-
`
|
|
418
|
-
`closed: ${issues.filter((i) => i.state !== "open").length}`,
|
|
388
|
+
`total: ${releases.length + (blogReleases?.length ?? 0)}`,
|
|
389
|
+
`latest: ${releases[0]?.tag || "unknown"}`,
|
|
419
390
|
"---"
|
|
420
391
|
].join("\n"),
|
|
421
392
|
"",
|
|
422
|
-
"#
|
|
393
|
+
"# Releases Index",
|
|
423
394
|
""
|
|
424
395
|
];
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
const date = isoDate(
|
|
434
|
-
|
|
396
|
+
if (blogReleases && blogReleases.length > 0) {
|
|
397
|
+
lines.push("## Blog Releases", "");
|
|
398
|
+
for (const b of blogReleases) lines.push(`- [${b.version}](./blog-${b.version}.md): ${b.title} (${b.date})`);
|
|
399
|
+
lines.push("");
|
|
400
|
+
}
|
|
401
|
+
if (releases.length > 0) {
|
|
402
|
+
if (blogReleases && blogReleases.length > 0) lines.push("## Release Notes", "");
|
|
403
|
+
for (const r of releases) {
|
|
404
|
+
const date = isoDate(r.publishedAt || r.createdAt);
|
|
405
|
+
const filename = r.tag.includes("@") || r.tag.startsWith("v") ? r.tag : `v${r.tag}`;
|
|
406
|
+
const sv = parseSemver(extractVersion(r.tag, pkg) || r.tag);
|
|
407
|
+
const label = sv?.patch === 0 && sv.minor === 0 ? " **[MAJOR]**" : sv?.patch === 0 ? " **[MINOR]**" : "";
|
|
408
|
+
lines.push(`- [${r.tag}](./${filename}.md): ${r.name || r.tag} (${date})${label}`);
|
|
435
409
|
}
|
|
436
|
-
|
|
410
|
+
lines.push("");
|
|
411
|
+
}
|
|
412
|
+
if (hasChangelog) {
|
|
413
|
+
lines.push("## Changelog", "");
|
|
414
|
+
lines.push("- [CHANGELOG.md](./CHANGELOG.md)");
|
|
415
|
+
lines.push("");
|
|
416
|
+
}
|
|
417
|
+
return lines.join("\n");
|
|
418
|
+
}
|
|
419
|
+
/**
|
|
420
|
+
* Check if a single release is a stub redirecting to CHANGELOG.md.
|
|
421
|
+
* Short body (<500 chars) that mentions CHANGELOG indicates no real content.
|
|
422
|
+
*/
|
|
423
|
+
function isStubRelease(release) {
|
|
424
|
+
const body = (release.markdown || "").trim();
|
|
425
|
+
return body.length < 500 && /changelog\.md/i.test(body);
|
|
426
|
+
}
|
|
427
|
+
/**
|
|
428
|
+
* Fetch CHANGELOG.md from a GitHub repo at a specific ref as fallback.
|
|
429
|
+
* For monorepos, also checks packages/{shortName}/CHANGELOG.md.
|
|
430
|
+
*/
|
|
431
|
+
async function fetchChangelog(owner, repo, ref, packageName) {
|
|
432
|
+
const paths = [];
|
|
433
|
+
if (packageName) {
|
|
434
|
+
const shortName = packageName.replace(/^@.*\//, "");
|
|
435
|
+
const scopeless = packageName.replace(/^@/, "").replace("/", "-");
|
|
436
|
+
const candidates = [...new Set([shortName, scopeless])];
|
|
437
|
+
for (const name of candidates) paths.push(`packages/${name}/CHANGELOG.md`);
|
|
438
|
+
}
|
|
439
|
+
paths.push("CHANGELOG.md", "changelog.md", "CHANGES.md");
|
|
440
|
+
for (const path of paths) {
|
|
441
|
+
const content = await fetchGitHubRaw(`https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${path}`);
|
|
442
|
+
if (content) return content;
|
|
443
|
+
}
|
|
444
|
+
return null;
|
|
445
|
+
}
|
|
446
|
+
/**
|
|
447
|
+
* Fetch release notes for a package. Returns CachedDoc[] with releases/{tag}.md files.
|
|
448
|
+
*
|
|
449
|
+
* Strategy:
|
|
450
|
+
* 1. Fetch GitHub releases, filter to package-specific tags for monorepos
|
|
451
|
+
* 2. If no releases found, try CHANGELOG.md as fallback
|
|
452
|
+
*/
|
|
453
|
+
async function fetchReleaseNotes(owner, repo, installedVersion, gitRef, packageName, fromDate, changelogRef) {
|
|
454
|
+
const selected = selectReleases(await fetchAllReleases(owner, repo), packageName, installedVersion, fromDate);
|
|
455
|
+
if (selected.length > 0) {
|
|
456
|
+
const docs = selected.filter((r) => !isStubRelease(r)).map((r) => {
|
|
457
|
+
return {
|
|
458
|
+
path: `releases/${r.tag.includes("@") || r.tag.startsWith("v") ? r.tag : `v${r.tag}`}.md`,
|
|
459
|
+
content: formatRelease(r, packageName)
|
|
460
|
+
};
|
|
461
|
+
});
|
|
462
|
+
const changelog = await fetchChangelog(owner, repo, changelogRef || gitRef || selected[0].tag, packageName);
|
|
463
|
+
if (changelog && changelog.length < 5e5) docs.push({
|
|
464
|
+
path: "releases/CHANGELOG.md",
|
|
465
|
+
content: changelog
|
|
466
|
+
});
|
|
467
|
+
return docs;
|
|
437
468
|
}
|
|
438
|
-
|
|
469
|
+
const changelog = await fetchChangelog(owner, repo, changelogRef || gitRef || "main", packageName);
|
|
470
|
+
if (!changelog) return [];
|
|
471
|
+
return [{
|
|
472
|
+
path: "releases/CHANGELOG.md",
|
|
473
|
+
content: changelog
|
|
474
|
+
}];
|
|
439
475
|
}
|
|
440
476
|
//#endregion
|
|
441
|
-
//#region src/sources/
|
|
442
|
-
/**
|
|
443
|
-
* Shared utilities for doc resolution
|
|
444
|
-
*/
|
|
445
|
-
const $fetch = ofetch.create({
|
|
446
|
-
retry: 3,
|
|
447
|
-
retryDelay: 500,
|
|
448
|
-
timeout: 15e3,
|
|
449
|
-
headers: { "User-Agent": "skilld/1.0" }
|
|
450
|
-
});
|
|
451
|
-
/**
|
|
452
|
-
* Fetch text content from URL
|
|
453
|
-
*/
|
|
454
|
-
async function fetchText(url) {
|
|
455
|
-
return $fetch(url, { responseType: "text" }).catch(() => null);
|
|
456
|
-
}
|
|
477
|
+
//#region src/sources/blog-releases.ts
|
|
457
478
|
/**
|
|
458
|
-
*
|
|
479
|
+
* Format a blog release as markdown with YAML frontmatter
|
|
459
480
|
*/
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
481
|
+
function formatBlogRelease(release) {
|
|
482
|
+
return `${[
|
|
483
|
+
"---",
|
|
484
|
+
`version: ${release.version}`,
|
|
485
|
+
`title: "${release.title.replace(/"/g, "\\\"")}"`,
|
|
486
|
+
`date: ${release.date}`,
|
|
487
|
+
`url: ${release.url}`,
|
|
488
|
+
`source: blog-release`,
|
|
489
|
+
"---"
|
|
490
|
+
].join("\n")}\n\n# ${release.title}\n\n${release.markdown}`;
|
|
464
491
|
}
|
|
465
492
|
/**
|
|
466
|
-
*
|
|
493
|
+
* Fetch and parse a single blog post using preset metadata for version/date
|
|
467
494
|
*/
|
|
468
|
-
|
|
469
|
-
"twitter.com",
|
|
470
|
-
"x.com",
|
|
471
|
-
"facebook.com",
|
|
472
|
-
"linkedin.com",
|
|
473
|
-
"youtube.com",
|
|
474
|
-
"instagram.com",
|
|
475
|
-
"npmjs.com",
|
|
476
|
-
"www.npmjs.com",
|
|
477
|
-
"yarnpkg.com"
|
|
478
|
-
]);
|
|
479
|
-
function isUselessDocsUrl(url) {
|
|
495
|
+
async function fetchBlogPost(entry) {
|
|
480
496
|
try {
|
|
481
|
-
const
|
|
482
|
-
|
|
497
|
+
const html = await $fetch(entry.url, {
|
|
498
|
+
responseType: "text",
|
|
499
|
+
signal: AbortSignal.timeout(1e4)
|
|
500
|
+
}).catch(() => null);
|
|
501
|
+
if (!html) return null;
|
|
502
|
+
let title = "";
|
|
503
|
+
const titleMatch = html.match(/<h1[^>]*>([^<]+)<\/h1>/);
|
|
504
|
+
if (titleMatch) title = titleMatch[1].trim();
|
|
505
|
+
if (!title) {
|
|
506
|
+
const metaTitleMatch = html.match(/<title>([^<]+)<\/title>/);
|
|
507
|
+
if (metaTitleMatch) title = metaTitleMatch[1].trim();
|
|
508
|
+
}
|
|
509
|
+
const markdown = htmlToMarkdown(html);
|
|
510
|
+
if (!markdown) return null;
|
|
511
|
+
return {
|
|
512
|
+
version: entry.version,
|
|
513
|
+
title: title || entry.title || `Release ${entry.version}`,
|
|
514
|
+
date: entry.date,
|
|
515
|
+
markdown,
|
|
516
|
+
url: entry.url
|
|
517
|
+
};
|
|
483
518
|
} catch {
|
|
484
|
-
return
|
|
519
|
+
return null;
|
|
485
520
|
}
|
|
486
521
|
}
|
|
487
522
|
/**
|
|
488
|
-
*
|
|
523
|
+
* Filter blog releases by installed version
|
|
524
|
+
* Only includes releases where version <= installedVersion
|
|
525
|
+
* Returns all releases if version parsing fails (fail-safe)
|
|
489
526
|
*/
|
|
490
|
-
function
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
return false;
|
|
496
|
-
|
|
527
|
+
function filterBlogsByVersion(entries, installedVersion) {
|
|
528
|
+
const installedSv = parseSemver(installedVersion);
|
|
529
|
+
if (!installedSv) return entries;
|
|
530
|
+
return entries.filter((entry) => {
|
|
531
|
+
const entrySv = parseSemver(entry.version);
|
|
532
|
+
if (!entrySv) return false;
|
|
533
|
+
return compareSemver(entrySv, installedSv) <= 0;
|
|
534
|
+
});
|
|
497
535
|
}
|
|
498
536
|
/**
|
|
499
|
-
*
|
|
537
|
+
* Fetch blog release notes from package presets
|
|
538
|
+
* Filters to only releases matching or older than the installed version
|
|
539
|
+
* Returns CachedDoc[] with releases/blog-{version}.md files
|
|
500
540
|
*/
|
|
501
|
-
function
|
|
502
|
-
const
|
|
503
|
-
if (!
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
541
|
+
async function fetchBlogReleases(packageName, installedVersion) {
|
|
542
|
+
const preset = getBlogPreset(packageName);
|
|
543
|
+
if (!preset) return [];
|
|
544
|
+
const filteredReleases = filterBlogsByVersion(preset.releases, installedVersion);
|
|
545
|
+
if (filteredReleases.length === 0) return [];
|
|
546
|
+
const releases = [];
|
|
547
|
+
const batchSize = 3;
|
|
548
|
+
for (let i = 0; i < filteredReleases.length; i += batchSize) {
|
|
549
|
+
const batch = filteredReleases.slice(i, i + batchSize);
|
|
550
|
+
const results = await Promise.all(batch.map((entry) => fetchBlogPost(entry)));
|
|
551
|
+
for (const result of results) if (result) releases.push(result);
|
|
552
|
+
}
|
|
553
|
+
if (releases.length === 0) return [];
|
|
554
|
+
releases.sort((a, b) => {
|
|
555
|
+
const aVer = a.version.split(".").map(Number);
|
|
556
|
+
const bVer = b.version.split(".").map(Number);
|
|
557
|
+
for (let i = 0; i < Math.max(aVer.length, bVer.length); i++) {
|
|
558
|
+
const diff = (bVer[i] ?? 0) - (aVer[i] ?? 0);
|
|
559
|
+
if (diff !== 0) return diff;
|
|
560
|
+
}
|
|
561
|
+
return 0;
|
|
562
|
+
});
|
|
563
|
+
return releases.map((r) => ({
|
|
564
|
+
path: `releases/blog-${r.version}.md`,
|
|
565
|
+
content: formatBlogRelease(r)
|
|
566
|
+
}));
|
|
508
567
|
}
|
|
568
|
+
//#endregion
|
|
569
|
+
//#region src/sources/crawl.ts
|
|
509
570
|
/**
|
|
510
|
-
*
|
|
571
|
+
* Website crawl doc source — fetches docs by crawling a URL pattern
|
|
511
572
|
*/
|
|
512
|
-
function normalizeRepoUrl(url) {
|
|
513
|
-
return url.replace(/^git\+/, "").replace(/#.*$/, "").replace(/\.git$/, "").replace(/^git:\/\//, "https://").replace(/^ssh:\/\/git@github\.com/, "https://github.com").replace(/^git@github\.com:/, "https://github.com/");
|
|
514
|
-
}
|
|
515
573
|
/**
|
|
516
|
-
*
|
|
517
|
-
*
|
|
574
|
+
* Crawl a URL pattern and return docs as cached doc format.
|
|
575
|
+
* Uses HTTP crawler (no browser needed) with sitemap discovery + glob filtering.
|
|
576
|
+
*
|
|
577
|
+
* @param url - URL with optional glob pattern (e.g. 'https://example.com/docs/**')
|
|
578
|
+
* @param onProgress - Optional progress callback
|
|
579
|
+
* @param maxPages - Max pages to crawl (default 200)
|
|
518
580
|
*/
|
|
519
|
-
function
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
581
|
+
async function fetchCrawledDocs(url, onProgress, maxPages = 200) {
|
|
582
|
+
const outputDir = join(tmpdir(), "skilld-crawl", Date.now().toString());
|
|
583
|
+
onProgress?.(`Crawling ${url}`);
|
|
584
|
+
const userLang = getUserLang();
|
|
585
|
+
const foreignUrls = /* @__PURE__ */ new Set();
|
|
586
|
+
const doCrawl = () => crawlAndGenerate({
|
|
587
|
+
urls: [url],
|
|
588
|
+
outputDir,
|
|
589
|
+
driver: "http",
|
|
590
|
+
generateLlmsTxt: false,
|
|
591
|
+
generateIndividualMd: true,
|
|
592
|
+
maxRequestsPerCrawl: maxPages,
|
|
593
|
+
onPage: (page) => {
|
|
594
|
+
const lang = extractHtmlLang(page.html);
|
|
595
|
+
if (lang && !lang.startsWith("en") && !lang.startsWith(userLang)) foreignUrls.add(page.url);
|
|
528
596
|
}
|
|
529
|
-
|
|
597
|
+
}, (progress) => {
|
|
598
|
+
if (progress.crawling.status === "processing" && progress.crawling.total > 0) onProgress?.(`Crawling ${progress.crawling.processed}/${progress.crawling.total} pages`);
|
|
599
|
+
});
|
|
600
|
+
let results = await doCrawl().catch((err) => {
|
|
601
|
+
onProgress?.(`Crawl failed: ${err?.message || err}`);
|
|
602
|
+
return [];
|
|
603
|
+
});
|
|
604
|
+
if (results.length === 0) {
|
|
605
|
+
onProgress?.("Retrying crawl");
|
|
606
|
+
results = await doCrawl().catch(() => []);
|
|
530
607
|
}
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
608
|
+
rmSync(outputDir, {
|
|
609
|
+
recursive: true,
|
|
610
|
+
force: true
|
|
611
|
+
});
|
|
612
|
+
const docs = [];
|
|
613
|
+
let localeFiltered = 0;
|
|
614
|
+
for (const result of results) {
|
|
615
|
+
if (!result.success || !result.content) continue;
|
|
616
|
+
if (foreignUrls.has(result.url)) {
|
|
617
|
+
localeFiltered++;
|
|
618
|
+
continue;
|
|
619
|
+
}
|
|
620
|
+
const segments = (new URL(result.url).pathname.replace(/\/$/, "") || "/index").split("/").filter(Boolean);
|
|
621
|
+
if (isForeignPathPrefix(segments[0], userLang)) {
|
|
622
|
+
localeFiltered++;
|
|
623
|
+
continue;
|
|
624
|
+
}
|
|
625
|
+
const path = `docs/${segments.join("/")}.md`;
|
|
626
|
+
docs.push({
|
|
627
|
+
path,
|
|
628
|
+
content: result.content
|
|
629
|
+
});
|
|
630
|
+
}
|
|
631
|
+
if (localeFiltered > 0) onProgress?.(`Filtered ${localeFiltered} foreign locale pages`);
|
|
632
|
+
onProgress?.(`Crawled ${docs.length} pages`);
|
|
633
|
+
return docs;
|
|
634
|
+
}
|
|
635
|
+
const HTML_LANG_RE = /<html[^>]*\slang=["']([^"']+)["']/i;
|
|
636
|
+
/** Extract lang attribute from <html> tag */
|
|
637
|
+
function extractHtmlLang(html) {
|
|
638
|
+
return HTML_LANG_RE.exec(html)?.[1]?.toLowerCase();
|
|
639
|
+
}
|
|
640
|
+
/** Common ISO 639-1 locale codes for i18n'd doc sites */
|
|
641
|
+
const LOCALE_CODES = new Set([
|
|
642
|
+
"ar",
|
|
643
|
+
"de",
|
|
644
|
+
"es",
|
|
645
|
+
"fr",
|
|
646
|
+
"id",
|
|
647
|
+
"it",
|
|
648
|
+
"ja",
|
|
649
|
+
"ko",
|
|
650
|
+
"nl",
|
|
651
|
+
"pl",
|
|
652
|
+
"pt",
|
|
653
|
+
"pt-br",
|
|
654
|
+
"ru",
|
|
655
|
+
"th",
|
|
656
|
+
"tr",
|
|
657
|
+
"uk",
|
|
658
|
+
"vi",
|
|
659
|
+
"zh",
|
|
660
|
+
"zh-cn",
|
|
661
|
+
"zh-tw"
|
|
662
|
+
]);
|
|
663
|
+
/** Check if a URL path segment is a known locale prefix foreign to both English and user's locale */
|
|
664
|
+
function isForeignPathPrefix(segment, userLang) {
|
|
665
|
+
if (!segment) return false;
|
|
666
|
+
const lower = segment.toLowerCase();
|
|
667
|
+
if (lower === "en" || lower.startsWith(userLang)) return false;
|
|
668
|
+
return LOCALE_CODES.has(lower);
|
|
537
669
|
}
|
|
538
|
-
/**
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
const hash = url.indexOf("#");
|
|
543
|
-
if (hash === -1) return void 0;
|
|
544
|
-
const fragment = url.slice(hash + 1);
|
|
545
|
-
if (!fragment || fragment === "readme") return void 0;
|
|
546
|
-
return fragment;
|
|
670
|
+
/** Detect user's 2-letter language code from env (e.g. 'ja' from LANG=ja_JP.UTF-8) */
|
|
671
|
+
function getUserLang() {
|
|
672
|
+
const code = (process.env.LC_ALL || process.env.LANG || process.env.LANGUAGE || "").split(/[_.:-]/)[0]?.toLowerCase() || "";
|
|
673
|
+
return code.length >= 2 ? code.slice(0, 2) : "en";
|
|
547
674
|
}
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
* GitHub release notes fetching via gh CLI (preferred) with ungh.cc fallback
|
|
552
|
-
*/
|
|
553
|
-
function parseSemver(version) {
|
|
554
|
-
const clean = version.replace(/^v/, "");
|
|
555
|
-
const match = clean.match(/^(\d+)(?:\.(\d+))?(?:\.(\d+))?/);
|
|
556
|
-
if (!match) return null;
|
|
557
|
-
return {
|
|
558
|
-
major: +match[1],
|
|
559
|
-
minor: match[2] ? +match[2] : 0,
|
|
560
|
-
patch: match[3] ? +match[3] : 0,
|
|
561
|
-
raw: clean
|
|
562
|
-
};
|
|
675
|
+
/** Append glob pattern to a docs URL for crawling */
|
|
676
|
+
function toCrawlPattern(docsUrl) {
|
|
677
|
+
return `${docsUrl.replace(/\/+$/, "")}/**`;
|
|
563
678
|
}
|
|
679
|
+
//#endregion
|
|
680
|
+
//#region src/sources/issues.ts
|
|
564
681
|
/**
|
|
565
|
-
*
|
|
566
|
-
* -
|
|
567
|
-
*
|
|
568
|
-
* - `v1.2.3` → `1.2.3`
|
|
569
|
-
* - `1.2.3` → `1.2.3`
|
|
682
|
+
* GitHub issues fetching via gh CLI Search API
|
|
683
|
+
* Freshness-weighted scoring, type quotas, comment quality filtering
|
|
684
|
+
* Categorized by labels, noise filtered out, non-technical issues detected
|
|
570
685
|
*/
|
|
571
|
-
|
|
572
|
-
if (packageName) {
|
|
573
|
-
const atMatch = tag.match(new RegExp(`^${escapeRegex(packageName)}@(.+)$`));
|
|
574
|
-
if (atMatch) return atMatch[1];
|
|
575
|
-
const dashMatch = tag.match(new RegExp(`^${escapeRegex(packageName)}-v?(.+)$`));
|
|
576
|
-
if (dashMatch) return dashMatch[1];
|
|
577
|
-
}
|
|
578
|
-
return tag.replace(/^v/, "");
|
|
579
|
-
}
|
|
580
|
-
function escapeRegex(str) {
|
|
581
|
-
return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
582
|
-
}
|
|
686
|
+
let _ghAvailable;
|
|
583
687
|
/**
|
|
584
|
-
* Check if
|
|
688
|
+
* Check if gh CLI is installed and authenticated (cached)
|
|
585
689
|
*/
|
|
586
|
-
function
|
|
587
|
-
|
|
690
|
+
function isGhAvailable() {
|
|
691
|
+
if (_ghAvailable !== void 0) return _ghAvailable;
|
|
692
|
+
const { status } = spawnSync("gh", ["auth", "status"], { stdio: "ignore" });
|
|
693
|
+
return _ghAvailable = status === 0;
|
|
588
694
|
}
|
|
695
|
+
/** Labels that indicate noise — filter these out entirely */
|
|
696
|
+
const NOISE_LABELS = new Set([
|
|
697
|
+
"duplicate",
|
|
698
|
+
"stale",
|
|
699
|
+
"invalid",
|
|
700
|
+
"wontfix",
|
|
701
|
+
"won't fix",
|
|
702
|
+
"spam",
|
|
703
|
+
"off-topic",
|
|
704
|
+
"needs triage",
|
|
705
|
+
"triage"
|
|
706
|
+
]);
|
|
707
|
+
/** Labels that indicate feature requests — deprioritize */
|
|
708
|
+
const FEATURE_LABELS = new Set([
|
|
709
|
+
"enhancement",
|
|
710
|
+
"feature",
|
|
711
|
+
"feature request",
|
|
712
|
+
"feature-request",
|
|
713
|
+
"proposal",
|
|
714
|
+
"rfc",
|
|
715
|
+
"idea",
|
|
716
|
+
"suggestion"
|
|
717
|
+
]);
|
|
718
|
+
const BUG_LABELS = new Set([
|
|
719
|
+
"bug",
|
|
720
|
+
"defect",
|
|
721
|
+
"regression",
|
|
722
|
+
"error",
|
|
723
|
+
"crash",
|
|
724
|
+
"fix",
|
|
725
|
+
"confirmed",
|
|
726
|
+
"verified"
|
|
727
|
+
]);
|
|
728
|
+
const QUESTION_LABELS = new Set([
|
|
729
|
+
"question",
|
|
730
|
+
"help wanted",
|
|
731
|
+
"support",
|
|
732
|
+
"usage",
|
|
733
|
+
"how-to",
|
|
734
|
+
"help",
|
|
735
|
+
"assistance"
|
|
736
|
+
]);
|
|
737
|
+
const DOCS_LABELS = new Set([
|
|
738
|
+
"documentation",
|
|
739
|
+
"docs",
|
|
740
|
+
"doc",
|
|
741
|
+
"typo"
|
|
742
|
+
]);
|
|
589
743
|
/**
|
|
590
|
-
* Check if a
|
|
744
|
+
* Check if a label contains any keyword from a set.
|
|
745
|
+
* Handles emoji-prefixed labels like ":sparkles: feature request" or ":lady_beetle: bug".
|
|
591
746
|
*/
|
|
592
|
-
function
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
function compareSemver(a, b) {
|
|
596
|
-
if (a.major !== b.major) return a.major - b.major;
|
|
597
|
-
if (a.minor !== b.minor) return a.minor - b.minor;
|
|
598
|
-
return a.patch - b.patch;
|
|
747
|
+
function labelMatchesAny(label, keywords) {
|
|
748
|
+
for (const keyword of keywords) if (label === keyword || label.includes(keyword)) return true;
|
|
749
|
+
return false;
|
|
599
750
|
}
|
|
600
751
|
/**
|
|
601
|
-
*
|
|
752
|
+
* Classify an issue by its labels into a type useful for skill generation
|
|
602
753
|
*/
|
|
603
|
-
function
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
".[] | {id: .id, tag: .tag_name, name: .name, prerelease: .prerelease, createdAt: .created_at, publishedAt: .published_at, markdown: .body}"
|
|
611
|
-
], {
|
|
612
|
-
encoding: "utf-8",
|
|
613
|
-
timeout: 3e4,
|
|
614
|
-
stdio: [
|
|
615
|
-
"ignore",
|
|
616
|
-
"pipe",
|
|
617
|
-
"ignore"
|
|
618
|
-
]
|
|
619
|
-
});
|
|
620
|
-
if (!ndjson) return [];
|
|
621
|
-
return ndjson.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line));
|
|
622
|
-
} catch {
|
|
623
|
-
return [];
|
|
624
|
-
}
|
|
754
|
+
function classifyIssue(labels) {
|
|
755
|
+
const lower = labels.map((l) => l.toLowerCase());
|
|
756
|
+
if (lower.some((l) => labelMatchesAny(l, BUG_LABELS))) return "bug";
|
|
757
|
+
if (lower.some((l) => labelMatchesAny(l, QUESTION_LABELS))) return "question";
|
|
758
|
+
if (lower.some((l) => labelMatchesAny(l, DOCS_LABELS))) return "docs";
|
|
759
|
+
if (lower.some((l) => labelMatchesAny(l, FEATURE_LABELS))) return "feature";
|
|
760
|
+
return "other";
|
|
625
761
|
}
|
|
626
762
|
/**
|
|
627
|
-
*
|
|
763
|
+
* Check if an issue should be filtered out entirely
|
|
628
764
|
*/
|
|
629
|
-
|
|
630
|
-
|
|
765
|
+
function isNoiseIssue(issue) {
|
|
766
|
+
if (issue.labels.map((l) => l.toLowerCase()).some((l) => labelMatchesAny(l, NOISE_LABELS))) return true;
|
|
767
|
+
if (issue.title.startsWith("☂️") || issue.title.startsWith("[META]") || issue.title.startsWith("[Tracking]")) return true;
|
|
768
|
+
return false;
|
|
631
769
|
}
|
|
632
|
-
/**
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
async function fetchAllReleases(owner, repo) {
|
|
636
|
-
if (isGhAvailable()) {
|
|
637
|
-
const releases = fetchReleasesViaGh(owner, repo);
|
|
638
|
-
if (releases.length > 0) return releases;
|
|
639
|
-
}
|
|
640
|
-
return fetchReleasesViaUngh(owner, repo);
|
|
770
|
+
/** Check if body contains a code block */
|
|
771
|
+
function hasCodeBlock$1(text) {
|
|
772
|
+
return /```[\s\S]*?```/.test(text) || /`[^`]+`/.test(text);
|
|
641
773
|
}
|
|
642
774
|
/**
|
|
643
|
-
*
|
|
644
|
-
*
|
|
645
|
-
*
|
|
646
|
-
* If installedVersion is provided, filters out releases newer than it.
|
|
775
|
+
* Detect non-technical issues: fan mail, showcases, sentiment.
|
|
776
|
+
* Short body + no code + high reactions = likely non-technical.
|
|
777
|
+
* Note: roadmap/tracking issues are NOT filtered — they get score-boosted instead.
|
|
647
778
|
*/
|
|
648
|
-
function
|
|
649
|
-
const
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
const sorted = releases.filter((r) => {
|
|
654
|
-
const ver = extractVersion(r.tag, hasMonorepoTags ? packageName : void 0);
|
|
655
|
-
if (!ver) return false;
|
|
656
|
-
const sv = parseSemver(ver);
|
|
657
|
-
if (!sv) return false;
|
|
658
|
-
if (hasMonorepoTags && packageName && !tagMatchesPackage(r.tag, packageName)) return false;
|
|
659
|
-
if (fromTs) {
|
|
660
|
-
const pubDate = r.publishedAt || r.createdAt;
|
|
661
|
-
if (pubDate && new Date(pubDate).getTime() < fromTs) return false;
|
|
662
|
-
}
|
|
663
|
-
if (r.prerelease) {
|
|
664
|
-
if (!installedIsPrerelease || !installedSv) return false;
|
|
665
|
-
return sv.major === installedSv.major && sv.minor === installedSv.minor;
|
|
666
|
-
}
|
|
667
|
-
if (installedSv && compareSemver(sv, installedSv) > 0) return false;
|
|
668
|
-
return true;
|
|
669
|
-
}).sort((a, b) => {
|
|
670
|
-
const verA = extractVersion(a.tag, hasMonorepoTags ? packageName : void 0);
|
|
671
|
-
const verB = extractVersion(b.tag, hasMonorepoTags ? packageName : void 0);
|
|
672
|
-
if (!verA || !verB) return 0;
|
|
673
|
-
return compareSemver(parseSemver(verB), parseSemver(verA));
|
|
674
|
-
});
|
|
675
|
-
return fromDate ? sorted : sorted.slice(0, 20);
|
|
779
|
+
function isNonTechnical(issue) {
|
|
780
|
+
const body = (issue.body || "").trim();
|
|
781
|
+
if (body.length < 200 && !hasCodeBlock$1(body) && issue.reactions > 50) return true;
|
|
782
|
+
if (/\b(?:love|thank|awesome|great work)\b/i.test(issue.title) && !hasCodeBlock$1(body)) return true;
|
|
783
|
+
return false;
|
|
676
784
|
}
|
|
677
785
|
/**
|
|
678
|
-
*
|
|
786
|
+
* Freshness-weighted score: reactions * decay(age_in_years)
|
|
787
|
+
* Steep decay so recent issues dominate over old high-reaction ones.
|
|
788
|
+
* At 0.6: 1yr=0.63x, 2yr=0.45x, 4yr=0.29x, 6yr=0.22x
|
|
679
789
|
*/
|
|
680
|
-
function
|
|
681
|
-
|
|
682
|
-
const version = extractVersion(release.tag, packageName) || release.tag;
|
|
683
|
-
const fm = [
|
|
684
|
-
"---",
|
|
685
|
-
`tag: ${release.tag}`,
|
|
686
|
-
`version: ${version}`,
|
|
687
|
-
`published: ${date}`
|
|
688
|
-
];
|
|
689
|
-
if (release.name && release.name !== release.tag) fm.push(`name: "${release.name.replace(/"/g, "\\\"")}"`);
|
|
690
|
-
fm.push("---");
|
|
691
|
-
return `${fm.join("\n")}\n\n# ${release.name || release.tag}\n\n${release.markdown}`;
|
|
790
|
+
function freshnessScore(reactions, createdAt) {
|
|
791
|
+
return reactions * (1 / (1 + (Date.now() - new Date(createdAt).getTime()) / (365.25 * 24 * 60 * 60 * 1e3) * .6));
|
|
692
792
|
}
|
|
693
793
|
/**
|
|
694
|
-
*
|
|
695
|
-
*
|
|
794
|
+
* Type quotas — guarantee a mix of issue types.
|
|
795
|
+
* Bugs and questions get priority; feature requests are hard-capped.
|
|
696
796
|
*/
|
|
697
|
-
function
|
|
698
|
-
const
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
[
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
lines.push("");
|
|
719
|
-
}
|
|
720
|
-
if (releases.length > 0) {
|
|
721
|
-
if (blogReleases && blogReleases.length > 0) lines.push("## Release Notes", "");
|
|
722
|
-
for (const r of releases) {
|
|
723
|
-
const date = isoDate(r.publishedAt || r.createdAt);
|
|
724
|
-
const filename = r.tag.includes("@") || r.tag.startsWith("v") ? r.tag : `v${r.tag}`;
|
|
725
|
-
const sv = parseSemver(extractVersion(r.tag, pkg) || r.tag);
|
|
726
|
-
const label = sv?.patch === 0 && sv.minor === 0 ? " **[MAJOR]**" : sv?.patch === 0 ? " **[MINOR]**" : "";
|
|
727
|
-
lines.push(`- [${r.tag}](./${filename}.md): ${r.name || r.tag} (${date})${label}`);
|
|
797
|
+
function applyTypeQuotas(issues, limit) {
|
|
798
|
+
const byType = /* @__PURE__ */ new Map();
|
|
799
|
+
for (const issue of issues) mapInsert(byType, issue.type, () => []).push(issue);
|
|
800
|
+
for (const group of byType.values()) group.sort((a, b) => b.score - a.score);
|
|
801
|
+
const quotas = [
|
|
802
|
+
["bug", Math.ceil(limit * .4)],
|
|
803
|
+
["question", Math.ceil(limit * .3)],
|
|
804
|
+
["docs", Math.ceil(limit * .15)],
|
|
805
|
+
["feature", Math.ceil(limit * .1)],
|
|
806
|
+
["other", Math.ceil(limit * .05)]
|
|
807
|
+
];
|
|
808
|
+
const selected = [];
|
|
809
|
+
const used = /* @__PURE__ */ new Set();
|
|
810
|
+
let remaining = limit;
|
|
811
|
+
for (const [type, quota] of quotas) {
|
|
812
|
+
const group = byType.get(type) || [];
|
|
813
|
+
const take = Math.min(quota, group.length, remaining);
|
|
814
|
+
for (let i = 0; i < take; i++) {
|
|
815
|
+
selected.push(group[i]);
|
|
816
|
+
used.add(group[i].number);
|
|
817
|
+
remaining--;
|
|
728
818
|
}
|
|
729
|
-
lines.push("");
|
|
730
819
|
}
|
|
731
|
-
if (
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
820
|
+
if (remaining > 0) {
|
|
821
|
+
const unused = issues.filter((i) => !used.has(i.number) && i.type !== "feature").sort((a, b) => b.score - a.score);
|
|
822
|
+
for (const issue of unused) {
|
|
823
|
+
if (remaining <= 0) break;
|
|
824
|
+
selected.push(issue);
|
|
825
|
+
remaining--;
|
|
826
|
+
}
|
|
735
827
|
}
|
|
736
|
-
return
|
|
828
|
+
return selected.sort((a, b) => b.score - a.score);
|
|
737
829
|
}
|
|
738
830
|
/**
|
|
739
|
-
*
|
|
740
|
-
* Short body (<500 chars) that mentions CHANGELOG indicates no real content.
|
|
831
|
+
* Body truncation limit based on reactions — high-reaction issues deserve more space
|
|
741
832
|
*/
|
|
742
|
-
function
|
|
743
|
-
|
|
744
|
-
|
|
833
|
+
function bodyLimit(reactions) {
|
|
834
|
+
if (reactions >= 10) return 2e3;
|
|
835
|
+
if (reactions >= 5) return 1500;
|
|
836
|
+
return 800;
|
|
745
837
|
}
|
|
746
838
|
/**
|
|
747
|
-
*
|
|
748
|
-
*
|
|
839
|
+
* Smart body truncation — preserves code blocks and error messages.
|
|
840
|
+
* Instead of slicing at a char limit, finds a safe break point.
|
|
749
841
|
*/
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
}).catch(() => null);
|
|
764
|
-
if (content) return content;
|
|
842
|
+
function truncateBody$1(body, limit) {
|
|
843
|
+
if (body.length <= limit) return body;
|
|
844
|
+
const codeBlockRe = /```[\s\S]*?```/g;
|
|
845
|
+
let lastSafeEnd = limit;
|
|
846
|
+
let match;
|
|
847
|
+
while ((match = codeBlockRe.exec(body)) !== null) {
|
|
848
|
+
const blockStart = match.index;
|
|
849
|
+
const blockEnd = blockStart + match[0].length;
|
|
850
|
+
if (blockStart < limit && blockEnd > limit) {
|
|
851
|
+
if (blockEnd <= limit + 500) lastSafeEnd = blockEnd;
|
|
852
|
+
else lastSafeEnd = blockStart;
|
|
853
|
+
break;
|
|
854
|
+
}
|
|
765
855
|
}
|
|
766
|
-
|
|
856
|
+
const slice = body.slice(0, lastSafeEnd);
|
|
857
|
+
const lastParagraph = slice.lastIndexOf("\n\n");
|
|
858
|
+
if (lastParagraph > lastSafeEnd * .6) return `${slice.slice(0, lastParagraph)}\n\n...`;
|
|
859
|
+
return `${slice}...`;
|
|
767
860
|
}
|
|
768
861
|
/**
|
|
769
|
-
* Fetch
|
|
770
|
-
*
|
|
771
|
-
* Strategy:
|
|
772
|
-
* 1. Fetch GitHub releases, filter to package-specific tags for monorepos
|
|
773
|
-
* 2. If no releases found, try CHANGELOG.md as fallback
|
|
862
|
+
* Fetch issues for a state using GitHub Search API sorted by reactions
|
|
774
863
|
*/
|
|
775
|
-
|
|
776
|
-
const
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
});
|
|
789
|
-
return docs;
|
|
864
|
+
function fetchIssuesByState(owner, repo, state, count, releasedAt, fromDate) {
|
|
865
|
+
const fetchCount = Math.min(count * 3, 100);
|
|
866
|
+
let datePart = "";
|
|
867
|
+
if (fromDate) datePart = state === "closed" ? `+closed:>=${fromDate}` : `+created:>=${fromDate}`;
|
|
868
|
+
else if (state === "closed") if (releasedAt) {
|
|
869
|
+
const date = new Date(releasedAt);
|
|
870
|
+
date.setMonth(date.getMonth() + 6);
|
|
871
|
+
datePart = `+closed:<=${isoDate(date.toISOString())}`;
|
|
872
|
+
} else datePart = `+closed:>${oneYearAgo()}`;
|
|
873
|
+
else if (releasedAt) {
|
|
874
|
+
const date = new Date(releasedAt);
|
|
875
|
+
date.setMonth(date.getMonth() + 6);
|
|
876
|
+
datePart = `+created:<=${isoDate(date.toISOString())}`;
|
|
790
877
|
}
|
|
791
|
-
const
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
878
|
+
const { stdout: result } = spawnSync("gh", [
|
|
879
|
+
"api",
|
|
880
|
+
`search/issues?q=${`repo:${owner}/${repo}+is:issue+is:${state}${datePart}`}&sort=reactions&order=desc&per_page=${fetchCount}`,
|
|
881
|
+
"-q",
|
|
882
|
+
".items[] | {number, title, state, labels: [.labels[]?.name], body, createdAt: .created_at, url: .html_url, reactions: .reactions[\"+1\"], comments: .comments, user: .user.login, userType: .user.type, authorAssociation: .author_association}"
|
|
883
|
+
], {
|
|
884
|
+
encoding: "utf-8",
|
|
885
|
+
maxBuffer: 10 * 1024 * 1024
|
|
886
|
+
});
|
|
887
|
+
if (!result) return [];
|
|
888
|
+
return result.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)).filter((issue) => !BOT_USERS.has(issue.user) && issue.userType !== "Bot").filter((issue) => !isNoiseIssue(issue)).filter((issue) => !isNonTechnical(issue)).map(({ user: _, userType: __, authorAssociation, ...issue }) => {
|
|
889
|
+
const isMaintainer = [
|
|
890
|
+
"OWNER",
|
|
891
|
+
"MEMBER",
|
|
892
|
+
"COLLABORATOR"
|
|
893
|
+
].includes(authorAssociation);
|
|
894
|
+
const isRoadmap = /\broadmap\b/i.test(issue.title) || issue.labels.some((l) => /roadmap/i.test(l));
|
|
895
|
+
return {
|
|
896
|
+
...issue,
|
|
897
|
+
type: classifyIssue(issue.labels),
|
|
898
|
+
topComments: [],
|
|
899
|
+
score: freshnessScore(issue.reactions, issue.createdAt) * (isMaintainer && isRoadmap ? 5 : 1)
|
|
900
|
+
};
|
|
901
|
+
}).sort((a, b) => b.score - a.score).slice(0, count);
|
|
797
902
|
}
|
|
798
|
-
|
|
799
|
-
|
|
903
|
+
function oneYearAgo() {
|
|
904
|
+
const d = /* @__PURE__ */ new Date();
|
|
905
|
+
d.setFullYear(d.getFullYear() - 1);
|
|
906
|
+
return isoDate(d.toISOString());
|
|
907
|
+
}
|
|
908
|
+
/** Noise patterns in comments — filter these out */
|
|
909
|
+
const COMMENT_NOISE_RE$1 = /^(?:\+1|👍|same here|any update|bump|following|is there any progress|when will this|me too|i have the same|same issue)[\s!?.]*$/i;
|
|
800
910
|
/**
|
|
801
|
-
*
|
|
911
|
+
* Batch-fetch top comments for issues via GraphQL.
|
|
912
|
+
* Enriches the top N highest-score issues with their best comments.
|
|
913
|
+
* Prioritizes: comments with code blocks, from maintainers, with high reactions.
|
|
914
|
+
* Filters out "+1", "any updates?", "same here" noise.
|
|
802
915
|
*/
|
|
803
|
-
function
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
916
|
+
function enrichWithComments(owner, repo, issues, topN = 15) {
|
|
917
|
+
const worth = issues.filter((i) => i.comments > 0 && (i.type === "bug" || i.type === "question" || i.reactions >= 3)).sort((a, b) => b.score - a.score).slice(0, topN);
|
|
918
|
+
if (worth.length === 0) return;
|
|
919
|
+
const query = `query($owner: String!, $repo: String!) { repository(owner: $owner, name: $repo) { ${worth.map((issue, i) => `i${i}: issue(number: ${issue.number}) { comments(first: 10) { nodes { body author { login } authorAssociation reactions { totalCount } } } }`).join(" ")} } }`;
|
|
920
|
+
try {
|
|
921
|
+
const { stdout: result } = spawnSync("gh", [
|
|
922
|
+
"api",
|
|
923
|
+
"graphql",
|
|
924
|
+
"-f",
|
|
925
|
+
`query=${query}`,
|
|
926
|
+
"-f",
|
|
927
|
+
`owner=${owner}`,
|
|
928
|
+
"-f",
|
|
929
|
+
`repo=${repo}`
|
|
930
|
+
], {
|
|
931
|
+
encoding: "utf-8",
|
|
932
|
+
maxBuffer: 10 * 1024 * 1024
|
|
933
|
+
});
|
|
934
|
+
if (!result) return;
|
|
935
|
+
const repo_ = JSON.parse(result)?.data?.repository;
|
|
936
|
+
if (!repo_) return;
|
|
937
|
+
for (let i = 0; i < worth.length; i++) {
|
|
938
|
+
const nodes = repo_[`i${i}`]?.comments?.nodes;
|
|
939
|
+
if (!Array.isArray(nodes)) continue;
|
|
940
|
+
const issue = worth[i];
|
|
941
|
+
const comments = nodes.filter((c) => c.author && !BOT_USERS.has(c.author.login)).filter((c) => !COMMENT_NOISE_RE$1.test((c.body || "").trim())).map((c) => {
|
|
942
|
+
const isMaintainer = [
|
|
943
|
+
"OWNER",
|
|
944
|
+
"MEMBER",
|
|
945
|
+
"COLLABORATOR"
|
|
946
|
+
].includes(c.authorAssociation);
|
|
947
|
+
const body = c.body || "";
|
|
948
|
+
const reactions = c.reactions?.totalCount || 0;
|
|
949
|
+
const _score = (isMaintainer ? 3 : 1) * (hasCodeBlock$1(body) ? 2 : 1) * (1 + reactions);
|
|
950
|
+
return {
|
|
951
|
+
body,
|
|
952
|
+
author: c.author.login,
|
|
953
|
+
reactions,
|
|
954
|
+
isMaintainer,
|
|
955
|
+
_score
|
|
956
|
+
};
|
|
957
|
+
}).sort((a, b) => b._score - a._score);
|
|
958
|
+
issue.topComments = comments.slice(0, 3).map(({ _score: _, ...c }) => c);
|
|
959
|
+
if (issue.state === "closed") issue.resolvedIn = detectResolvedVersion(comments);
|
|
960
|
+
}
|
|
961
|
+
} catch {}
|
|
813
962
|
}
|
|
814
963
|
/**
|
|
815
|
-
*
|
|
964
|
+
* Try to detect which version fixed a closed issue from maintainer comments.
|
|
965
|
+
* Looks for version patterns in maintainer/collaborator comments.
|
|
816
966
|
*/
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
const titleMatch = html.match(/<h1[^>]*>([^<]+)<\/h1>/);
|
|
826
|
-
if (titleMatch) title = titleMatch[1].trim();
|
|
827
|
-
if (!title) {
|
|
828
|
-
const metaTitleMatch = html.match(/<title>([^<]+)<\/title>/);
|
|
829
|
-
if (metaTitleMatch) title = metaTitleMatch[1].trim();
|
|
967
|
+
function detectResolvedVersion(comments) {
|
|
968
|
+
const maintainerComments = comments.filter((c) => c.isMaintainer);
|
|
969
|
+
for (const c of maintainerComments.reverse()) {
|
|
970
|
+
const match = c.body.match(/(?:fixed|landed|released|available|shipped|resolved|included)\s+in\s+v?(\d+\.\d+(?:\.\d+)?)/i);
|
|
971
|
+
if (match) return match[1];
|
|
972
|
+
if (c.body.length < 100) {
|
|
973
|
+
const vMatch = c.body.match(/\bv?(\d+\.\d+\.\d+)\b/);
|
|
974
|
+
if (vMatch) return vMatch[1];
|
|
830
975
|
}
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
976
|
+
}
|
|
977
|
+
}
|
|
978
|
+
/**
|
|
979
|
+
* Fetch issues from a GitHub repo with freshness-weighted scoring and type quotas.
|
|
980
|
+
* Returns a balanced mix: bugs > questions > docs > other > features.
|
|
981
|
+
* Filters noise, non-technical content, and enriches with quality comments.
|
|
982
|
+
*/
|
|
983
|
+
async function fetchGitHubIssues(owner, repo, limit = 30, releasedAt, fromDate) {
|
|
984
|
+
if (!isGhAvailable()) return [];
|
|
985
|
+
const openCount = Math.ceil(limit * .75);
|
|
986
|
+
const closedCount = limit - openCount;
|
|
987
|
+
try {
|
|
988
|
+
const open = fetchIssuesByState(owner, repo, "open", Math.min(openCount * 2, 100), releasedAt, fromDate);
|
|
989
|
+
const closed = fetchIssuesByState(owner, repo, "closed", Math.min(closedCount * 2, 50), releasedAt, fromDate);
|
|
990
|
+
const selected = applyTypeQuotas([...open, ...closed], limit);
|
|
991
|
+
enrichWithComments(owner, repo, selected);
|
|
992
|
+
return selected;
|
|
840
993
|
} catch {
|
|
841
|
-
return
|
|
994
|
+
return [];
|
|
842
995
|
}
|
|
843
996
|
}
|
|
844
997
|
/**
|
|
845
|
-
*
|
|
846
|
-
* Only includes releases where version <= installedVersion
|
|
847
|
-
* Returns all releases if version parsing fails (fail-safe)
|
|
848
|
-
*/
|
|
849
|
-
function filterBlogsByVersion(entries, installedVersion) {
|
|
850
|
-
const installedSv = parseSemver(installedVersion);
|
|
851
|
-
if (!installedSv) return entries;
|
|
852
|
-
return entries.filter((entry) => {
|
|
853
|
-
const entrySv = parseSemver(entry.version);
|
|
854
|
-
if (!entrySv) return false;
|
|
855
|
-
return compareSemver(entrySv, installedSv) <= 0;
|
|
856
|
-
});
|
|
857
|
-
}
|
|
858
|
-
/**
|
|
859
|
-
* Fetch blog release notes from package presets
|
|
860
|
-
* Filters to only releases matching or older than the installed version
|
|
861
|
-
* Returns CachedDoc[] with releases/blog-{version}.md files
|
|
998
|
+
* Format a single issue as markdown with YAML frontmatter
|
|
862
999
|
*/
|
|
863
|
-
|
|
864
|
-
const
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
1000
|
+
function formatIssueAsMarkdown(issue) {
|
|
1001
|
+
const limit = bodyLimit(issue.reactions);
|
|
1002
|
+
const fmFields = {
|
|
1003
|
+
number: issue.number,
|
|
1004
|
+
title: issue.title,
|
|
1005
|
+
type: issue.type,
|
|
1006
|
+
state: issue.state,
|
|
1007
|
+
created: isoDate(issue.createdAt),
|
|
1008
|
+
url: issue.url,
|
|
1009
|
+
reactions: issue.reactions,
|
|
1010
|
+
comments: issue.comments
|
|
1011
|
+
};
|
|
1012
|
+
if (issue.resolvedIn) fmFields.resolvedIn = issue.resolvedIn;
|
|
1013
|
+
if (issue.labels.length > 0) fmFields.labels = `[${issue.labels.join(", ")}]`;
|
|
1014
|
+
const lines = [
|
|
1015
|
+
buildFrontmatter(fmFields),
|
|
1016
|
+
"",
|
|
1017
|
+
`# ${issue.title}`
|
|
1018
|
+
];
|
|
1019
|
+
if (issue.body) {
|
|
1020
|
+
const body = truncateBody$1(issue.body, limit);
|
|
1021
|
+
lines.push("", body);
|
|
874
1022
|
}
|
|
875
|
-
if (
|
|
876
|
-
|
|
877
|
-
const
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
const
|
|
881
|
-
|
|
1023
|
+
if (issue.topComments.length > 0) {
|
|
1024
|
+
lines.push("", "---", "", "## Top Comments");
|
|
1025
|
+
for (const c of issue.topComments) {
|
|
1026
|
+
const reactions = c.reactions > 0 ? ` (+${c.reactions})` : "";
|
|
1027
|
+
const maintainer = c.isMaintainer ? " [maintainer]" : "";
|
|
1028
|
+
const commentBody = truncateBody$1(c.body, 600);
|
|
1029
|
+
lines.push("", `**@${c.author}**${maintainer}${reactions}:`, "", commentBody);
|
|
882
1030
|
}
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
return releases.map((r) => ({
|
|
886
|
-
path: `releases/blog-${r.version}.md`,
|
|
887
|
-
content: formatBlogRelease(r)
|
|
888
|
-
}));
|
|
1031
|
+
}
|
|
1032
|
+
return lines.join("\n");
|
|
889
1033
|
}
|
|
890
|
-
//#endregion
|
|
891
|
-
//#region src/sources/crawl.ts
|
|
892
|
-
/**
|
|
893
|
-
* Website crawl doc source — fetches docs by crawling a URL pattern
|
|
894
|
-
*/
|
|
895
1034
|
/**
|
|
896
|
-
*
|
|
897
|
-
*
|
|
898
|
-
*
|
|
899
|
-
* @param url - URL with optional glob pattern (e.g. 'https://example.com/docs/**')
|
|
900
|
-
* @param onProgress - Optional progress callback
|
|
901
|
-
* @param maxPages - Max pages to crawl (default 200)
|
|
1035
|
+
* Generate a summary index of all issues for quick LLM scanning.
|
|
1036
|
+
* Groups by type so the LLM can quickly find bugs vs questions.
|
|
902
1037
|
*/
|
|
903
|
-
|
|
904
|
-
const
|
|
905
|
-
|
|
906
|
-
const
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
const segments = (new URL(result.url).pathname.replace(/\/$/, "") || "/index").split("/").filter(Boolean);
|
|
943
|
-
if (isForeignPathPrefix(segments[0], userLang)) {
|
|
944
|
-
localeFiltered++;
|
|
945
|
-
continue;
|
|
1038
|
+
function generateIssueIndex(issues) {
|
|
1039
|
+
const byType = /* @__PURE__ */ new Map();
|
|
1040
|
+
for (const issue of issues) mapInsert(byType, issue.type, () => []).push(issue);
|
|
1041
|
+
const typeLabels = {
|
|
1042
|
+
bug: "Bugs & Regressions",
|
|
1043
|
+
question: "Questions & Usage Help",
|
|
1044
|
+
docs: "Documentation",
|
|
1045
|
+
feature: "Feature Requests",
|
|
1046
|
+
other: "Other"
|
|
1047
|
+
};
|
|
1048
|
+
const typeOrder = [
|
|
1049
|
+
"bug",
|
|
1050
|
+
"question",
|
|
1051
|
+
"docs",
|
|
1052
|
+
"other",
|
|
1053
|
+
"feature"
|
|
1054
|
+
];
|
|
1055
|
+
const sections = [
|
|
1056
|
+
[
|
|
1057
|
+
"---",
|
|
1058
|
+
`total: ${issues.length}`,
|
|
1059
|
+
`open: ${issues.filter((i) => i.state === "open").length}`,
|
|
1060
|
+
`closed: ${issues.filter((i) => i.state !== "open").length}`,
|
|
1061
|
+
"---"
|
|
1062
|
+
].join("\n"),
|
|
1063
|
+
"",
|
|
1064
|
+
"# Issues Index",
|
|
1065
|
+
""
|
|
1066
|
+
];
|
|
1067
|
+
for (const type of typeOrder) {
|
|
1068
|
+
const group = byType.get(type);
|
|
1069
|
+
if (!group?.length) continue;
|
|
1070
|
+
sections.push(`## ${typeLabels[type]} (${group.length})`, "");
|
|
1071
|
+
for (const issue of group) {
|
|
1072
|
+
const reactions = issue.reactions > 0 ? ` (+${issue.reactions})` : "";
|
|
1073
|
+
const state = issue.state === "open" ? "" : " [closed]";
|
|
1074
|
+
const resolved = issue.resolvedIn ? ` [fixed in ${issue.resolvedIn}]` : "";
|
|
1075
|
+
const date = isoDate(issue.createdAt);
|
|
1076
|
+
sections.push(`- [#${issue.number}](./issue-${issue.number}.md): ${issue.title}${reactions}${state}${resolved} (${date})`);
|
|
946
1077
|
}
|
|
947
|
-
|
|
948
|
-
docs.push({
|
|
949
|
-
path,
|
|
950
|
-
content: result.content
|
|
951
|
-
});
|
|
1078
|
+
sections.push("");
|
|
952
1079
|
}
|
|
953
|
-
|
|
954
|
-
onProgress?.(`Crawled ${docs.length} pages`);
|
|
955
|
-
return docs;
|
|
956
|
-
}
|
|
957
|
-
const HTML_LANG_RE = /<html[^>]*\slang=["']([^"']+)["']/i;
|
|
958
|
-
/** Extract lang attribute from <html> tag */
|
|
959
|
-
function extractHtmlLang(html) {
|
|
960
|
-
return HTML_LANG_RE.exec(html)?.[1]?.toLowerCase();
|
|
961
|
-
}
|
|
962
|
-
/** Common ISO 639-1 locale codes for i18n'd doc sites */
|
|
963
|
-
const LOCALE_CODES = new Set([
|
|
964
|
-
"ar",
|
|
965
|
-
"de",
|
|
966
|
-
"es",
|
|
967
|
-
"fr",
|
|
968
|
-
"id",
|
|
969
|
-
"it",
|
|
970
|
-
"ja",
|
|
971
|
-
"ko",
|
|
972
|
-
"nl",
|
|
973
|
-
"pl",
|
|
974
|
-
"pt",
|
|
975
|
-
"pt-br",
|
|
976
|
-
"ru",
|
|
977
|
-
"th",
|
|
978
|
-
"tr",
|
|
979
|
-
"uk",
|
|
980
|
-
"vi",
|
|
981
|
-
"zh",
|
|
982
|
-
"zh-cn",
|
|
983
|
-
"zh-tw"
|
|
984
|
-
]);
|
|
985
|
-
/** Check if a URL path segment is a known locale prefix foreign to both English and user's locale */
|
|
986
|
-
function isForeignPathPrefix(segment, userLang) {
|
|
987
|
-
if (!segment) return false;
|
|
988
|
-
const lower = segment.toLowerCase();
|
|
989
|
-
if (lower === "en" || lower.startsWith(userLang)) return false;
|
|
990
|
-
return LOCALE_CODES.has(lower);
|
|
991
|
-
}
|
|
992
|
-
/** Detect user's 2-letter language code from env (e.g. 'ja' from LANG=ja_JP.UTF-8) */
|
|
993
|
-
function getUserLang() {
|
|
994
|
-
const code = (process.env.LC_ALL || process.env.LANG || process.env.LANGUAGE || "").split(/[_.:-]/)[0]?.toLowerCase() || "";
|
|
995
|
-
return code.length >= 2 ? code.slice(0, 2) : "en";
|
|
996
|
-
}
|
|
997
|
-
/** Append glob pattern to a docs URL for crawling */
|
|
998
|
-
function toCrawlPattern(docsUrl) {
|
|
999
|
-
return `${docsUrl.replace(/\/+$/, "")}/**`;
|
|
1080
|
+
return sections.join("\n");
|
|
1000
1081
|
}
|
|
1001
1082
|
//#endregion
|
|
1002
1083
|
//#region src/sources/discussions.ts
|
|
@@ -1506,7 +1587,8 @@ async function downloadGitHubSkills(owner, repo, ref, skillPath, onProgress) {
|
|
|
1506
1587
|
onProgress?.(`Downloading ${owner}/${repo}/${skillPath}@${ref}`);
|
|
1507
1588
|
const { dir } = await downloadTemplate(`github:${owner}/${repo}/${skillPath}#${ref}`, {
|
|
1508
1589
|
dir: tempDir,
|
|
1509
|
-
force: true
|
|
1590
|
+
force: true,
|
|
1591
|
+
auth: getGitHubToken() || void 0
|
|
1510
1592
|
});
|
|
1511
1593
|
const skill = readLocalSkill(dir, skillPath);
|
|
1512
1594
|
return skill ? [skill] : [];
|
|
@@ -1515,7 +1597,8 @@ async function downloadGitHubSkills(owner, repo, ref, skillPath, onProgress) {
|
|
|
1515
1597
|
try {
|
|
1516
1598
|
const { dir } = await downloadTemplate(`github:${owner}/${repo}/skills#${ref}`, {
|
|
1517
1599
|
dir: tempDir,
|
|
1518
|
-
force: true
|
|
1600
|
+
force: true,
|
|
1601
|
+
auth: getGitHubToken() || void 0
|
|
1519
1602
|
});
|
|
1520
1603
|
const skills = [];
|
|
1521
1604
|
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
@@ -1528,7 +1611,7 @@ async function downloadGitHubSkills(owner, repo, ref, skillPath, onProgress) {
|
|
|
1528
1611
|
return skills;
|
|
1529
1612
|
}
|
|
1530
1613
|
} catch {}
|
|
1531
|
-
const content = await
|
|
1614
|
+
const content = await fetchGitHubRaw(`https://raw.githubusercontent.com/${owner}/${repo}/${ref}/SKILL.md`);
|
|
1532
1615
|
if (content) {
|
|
1533
1616
|
const fm = parseSkillFrontmatterName(content);
|
|
1534
1617
|
onProgress?.("Found 1 skill");
|
|
@@ -1697,10 +1780,20 @@ const MIN_GIT_DOCS = 5;
|
|
|
1697
1780
|
/** True when git-docs exist but are too few to be useful (< MIN_GIT_DOCS) */
|
|
1698
1781
|
const isShallowGitDocs = (n) => n > 0 && n < 5;
|
|
1699
1782
|
/**
|
|
1700
|
-
* List files at a git ref
|
|
1783
|
+
* List files at a git ref. Tries ungh.cc first (fast, no rate limits),
|
|
1784
|
+
* falls back to GitHub API for private repos.
|
|
1701
1785
|
*/
|
|
1702
1786
|
async function listFilesAtRef(owner, repo, ref) {
|
|
1703
|
-
|
|
1787
|
+
if (!isKnownPrivateRepo(owner, repo)) {
|
|
1788
|
+
const data = await $fetch(`https://ungh.cc/repos/${owner}/${repo}/files/${ref}`).catch(() => null);
|
|
1789
|
+
if (data?.files?.length) return data.files.map((f) => f.path);
|
|
1790
|
+
}
|
|
1791
|
+
const tree = await ghApi(`repos/${owner}/${repo}/git/trees/${ref}?recursive=1`);
|
|
1792
|
+
if (tree?.tree?.length) {
|
|
1793
|
+
markRepoPrivate(owner, repo);
|
|
1794
|
+
return tree.tree.map((f) => f.path);
|
|
1795
|
+
}
|
|
1796
|
+
return [];
|
|
1704
1797
|
}
|
|
1705
1798
|
/**
|
|
1706
1799
|
* Find git tag for a version by checking if ungh can list files at that ref.
|
|
@@ -1738,13 +1831,29 @@ async function findGitTag(owner, repo, version, packageName, branchHint) {
|
|
|
1738
1831
|
return null;
|
|
1739
1832
|
}
|
|
1740
1833
|
/**
|
|
1741
|
-
*
|
|
1742
|
-
|
|
1834
|
+
* Fetch releases from ungh.cc first, fall back to GitHub API for private repos.
|
|
1835
|
+
*/
|
|
1836
|
+
async function fetchUnghReleases(owner, repo) {
|
|
1837
|
+
if (!isKnownPrivateRepo(owner, repo)) {
|
|
1838
|
+
const data = await $fetch(`https://ungh.cc/repos/${owner}/${repo}/releases`).catch(() => null);
|
|
1839
|
+
if (data?.releases?.length) return data.releases;
|
|
1840
|
+
}
|
|
1841
|
+
const raw = await ghApiPaginated(`repos/${owner}/${repo}/releases`);
|
|
1842
|
+
if (raw.length > 0) {
|
|
1843
|
+
markRepoPrivate(owner, repo);
|
|
1844
|
+
return raw.map((r) => ({
|
|
1845
|
+
tag: r.tag_name,
|
|
1846
|
+
publishedAt: r.published_at
|
|
1847
|
+
}));
|
|
1848
|
+
}
|
|
1849
|
+
return [];
|
|
1850
|
+
}
|
|
1851
|
+
/**
|
|
1852
|
+
* Find the latest release tag matching `{packageName}@*`.
|
|
1743
1853
|
*/
|
|
1744
1854
|
async function findLatestReleaseTag(owner, repo, packageName) {
|
|
1745
|
-
const data = await $fetch(`https://ungh.cc/repos/${owner}/${repo}/releases`).catch(() => null);
|
|
1746
1855
|
const prefix = `${packageName}@`;
|
|
1747
|
-
return
|
|
1856
|
+
return (await fetchUnghReleases(owner, repo)).find((r) => r.tag.startsWith(prefix))?.tag ?? null;
|
|
1748
1857
|
}
|
|
1749
1858
|
/**
|
|
1750
1859
|
* Filter file paths by prefix and md/mdx extension
|
|
@@ -1994,7 +2103,7 @@ async function verifyNpmRepo(owner, repo, packageName) {
|
|
|
1994
2103
|
`packages/${packageName.replace(/^@/, "").replace("/", "-")}/package.json`
|
|
1995
2104
|
];
|
|
1996
2105
|
for (const path of paths) {
|
|
1997
|
-
const text = await
|
|
2106
|
+
const text = await fetchGitHubRaw(`${base}/${path}`);
|
|
1998
2107
|
if (!text) continue;
|
|
1999
2108
|
try {
|
|
2000
2109
|
if (JSON.parse(text).name === packageName) return true;
|
|
@@ -2051,38 +2160,35 @@ async function searchGitHubRepo(packageName) {
|
|
|
2051
2160
|
async function fetchGitHubRepoMeta(owner, repo, packageName) {
|
|
2052
2161
|
const override = packageName ? getDocOverride(packageName) : void 0;
|
|
2053
2162
|
if (override?.homepage) return { homepage: override.homepage };
|
|
2054
|
-
|
|
2055
|
-
const { stdout: json } = spawnSync("gh", [
|
|
2056
|
-
"api",
|
|
2057
|
-
`repos/${owner}/${repo}`,
|
|
2058
|
-
"-q",
|
|
2059
|
-
"{homepage}"
|
|
2060
|
-
], {
|
|
2061
|
-
encoding: "utf-8",
|
|
2062
|
-
timeout: 1e4
|
|
2063
|
-
});
|
|
2064
|
-
if (!json) throw new Error("no output");
|
|
2065
|
-
const data = JSON.parse(json);
|
|
2066
|
-
return data?.homepage ? { homepage: data.homepage } : null;
|
|
2067
|
-
} catch {}
|
|
2068
|
-
const data = await $fetch(`https://api.github.com/repos/${owner}/${repo}`).catch(() => null);
|
|
2163
|
+
const data = await ghApi(`repos/${owner}/${repo}`) ?? await $fetch(`https://api.github.com/repos/${owner}/${repo}`).catch(() => null);
|
|
2069
2164
|
return data?.homepage ? { homepage: data.homepage } : null;
|
|
2070
2165
|
}
|
|
2071
2166
|
/**
|
|
2072
2167
|
* Resolve README URL for a GitHub repo, returns ungh:// pseudo-URL or raw URL
|
|
2073
2168
|
*/
|
|
2074
2169
|
async function fetchReadme(owner, repo, subdir, ref) {
|
|
2075
|
-
const
|
|
2076
|
-
if ((
|
|
2170
|
+
const branch = ref || "main";
|
|
2171
|
+
if (!isKnownPrivateRepo(owner, repo)) {
|
|
2172
|
+
const unghUrl = subdir ? `https://ungh.cc/repos/${owner}/${repo}/files/${branch}/${subdir}/README.md` : `https://ungh.cc/repos/${owner}/${repo}/readme${ref ? `?ref=${ref}` : ""}`;
|
|
2173
|
+
if ((await $fetch.raw(unghUrl).catch(() => null))?.ok) return `ungh://${owner}/${repo}${subdir ? `/${subdir}` : ""}${ref ? `@${ref}` : ""}`;
|
|
2174
|
+
}
|
|
2077
2175
|
const basePath = subdir ? `${subdir}/` : "";
|
|
2078
2176
|
const branches = ref ? [ref] : ["main", "master"];
|
|
2177
|
+
const token = isKnownPrivateRepo(owner, repo) ? getGitHubToken() : null;
|
|
2178
|
+
const authHeaders = token ? { Authorization: `token ${token}` } : {};
|
|
2079
2179
|
for (const b of branches) for (const filename of [
|
|
2080
2180
|
"README.md",
|
|
2081
2181
|
"Readme.md",
|
|
2082
2182
|
"readme.md"
|
|
2083
2183
|
]) {
|
|
2084
2184
|
const readmeUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${b}/${basePath}${filename}`;
|
|
2085
|
-
if ((await $fetch.raw(readmeUrl).catch(() => null))?.ok) return readmeUrl;
|
|
2185
|
+
if ((await $fetch.raw(readmeUrl, { headers: authHeaders }).catch(() => null))?.ok) return readmeUrl;
|
|
2186
|
+
}
|
|
2187
|
+
const refParam = ref ? `?ref=${ref}` : "";
|
|
2188
|
+
const apiData = await ghApi(subdir ? `repos/${owner}/${repo}/contents/${subdir}/README.md${refParam}` : `repos/${owner}/${repo}/readme${refParam}`);
|
|
2189
|
+
if (apiData?.download_url) {
|
|
2190
|
+
markRepoPrivate(owner, repo);
|
|
2191
|
+
return apiData.download_url;
|
|
2086
2192
|
}
|
|
2087
2193
|
return null;
|
|
2088
2194
|
}
|
|
@@ -2116,6 +2222,7 @@ async function fetchReadmeContent(url) {
|
|
|
2116
2222
|
return text;
|
|
2117
2223
|
}
|
|
2118
2224
|
}
|
|
2225
|
+
if (url.includes("raw.githubusercontent.com")) return fetchGitHubRaw(url);
|
|
2119
2226
|
return fetchText(url);
|
|
2120
2227
|
}
|
|
2121
2228
|
/**
|
|
@@ -2125,34 +2232,14 @@ async function fetchReadmeContent(url) {
|
|
|
2125
2232
|
async function resolveGitHubRepo(owner, repo, onProgress) {
|
|
2126
2233
|
onProgress?.("Fetching repo metadata");
|
|
2127
2234
|
const repoUrl = `https://github.com/${owner}/${repo}`;
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
const { stdout: json } = spawnSync("gh", [
|
|
2132
|
-
"api",
|
|
2133
|
-
`repos/${owner}/${repo}`,
|
|
2134
|
-
"--jq",
|
|
2135
|
-
"{homepage: .homepage, description: .description}"
|
|
2136
|
-
], {
|
|
2137
|
-
encoding: "utf-8",
|
|
2138
|
-
timeout: 1e4
|
|
2139
|
-
});
|
|
2140
|
-
if (json) {
|
|
2141
|
-
const data = JSON.parse(json);
|
|
2142
|
-
homepage = data.homepage || void 0;
|
|
2143
|
-
description = data.description || void 0;
|
|
2144
|
-
}
|
|
2145
|
-
} catch {}
|
|
2146
|
-
if (!homepage && !description) {
|
|
2147
|
-
const data = await $fetch(`https://api.github.com/repos/${owner}/${repo}`).catch(() => null);
|
|
2148
|
-
homepage = data?.homepage || void 0;
|
|
2149
|
-
description = data?.description || void 0;
|
|
2150
|
-
}
|
|
2235
|
+
const meta = await ghApi(`repos/${owner}/${repo}`) ?? await $fetch(`https://api.github.com/repos/${owner}/${repo}`).catch(() => null);
|
|
2236
|
+
const homepage = meta?.homepage || void 0;
|
|
2237
|
+
const description = meta?.description || void 0;
|
|
2151
2238
|
onProgress?.("Fetching latest release");
|
|
2152
|
-
const
|
|
2239
|
+
const releases = await fetchUnghReleases(owner, repo);
|
|
2153
2240
|
let version = "main";
|
|
2154
2241
|
let releasedAt;
|
|
2155
|
-
const latestRelease =
|
|
2242
|
+
const latestRelease = releases[0];
|
|
2156
2243
|
if (latestRelease) {
|
|
2157
2244
|
version = latestRelease.tag.replace(/^v/, "");
|
|
2158
2245
|
releasedAt = latestRelease.publishedAt;
|
|
@@ -2628,6 +2715,6 @@ function getInstalledSkillVersion(skillDir) {
|
|
|
2628
2715
|
return readFileSync(skillPath, "utf-8").match(/^version:\s*"?([^"\n]+)"?/m)?.[1] || null;
|
|
2629
2716
|
}
|
|
2630
2717
|
//#endregion
|
|
2631
|
-
export {
|
|
2718
|
+
export { isGitHubRepoUrl as $, parseGitSkillInput as A, isGhAvailable as B, downloadLlmsDocs as C, normalizeLlmsLinks as D, fetchLlmsUrl as E, formatDiscussionAsMarkdown as F, fetchReleaseNotes as G, toCrawlPattern as H, generateDiscussionIndex as I, parseSemver as J, generateReleaseIndex as K, fetchGitHubIssues as L, resolveEntryFiles as M, generateDocsIndex as N, parseMarkdownLinks as O, fetchGitHubDiscussions as P, fetchText as Q, formatIssueAsMarkdown as R, validateGitDocsWithLlms as S, fetchLlmsTxt as T, fetchBlogReleases as U, fetchCrawledDocs as V, compareSemver as W, extractBranchHint as X, $fetch as Y, fetchGitHubRaw as Z, fetchReadme as _, getInstalledSkillVersion as a, isShallowGitDocs as b, readLocalPackageInfo as c, resolvePackageDocs as d, normalizeRepoUrl as et, resolvePackageDocsWithAttempts as f, fetchGitHubRepoMeta as g, fetchGitDocs as h, fetchPkgDist as i, parseSkillFrontmatterName as j, fetchGitSkills as k, resolveInstalledVersion as l, MIN_GIT_DOCS as m, fetchNpmPackage as n, parsePackageSpec as nt, parseVersionSpecifier as o, searchNpmPackages as p, isPrerelease as q, fetchNpmRegistryMeta as r, verifyUrl as rt, readLocalDependencies as s, fetchLatestVersion as t, parseGitHubUrl as tt, resolveLocalPackageDocs as u, fetchReadmeContent as v, extractSections as w, resolveGitHubRepo as x, filterFrameworkDocs as y, generateIssueIndex as z };
|
|
2632
2719
|
|
|
2633
2720
|
//# sourceMappingURL=sources.mjs.map
|