skilld 1.1.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_chunks/agent.mjs +13 -1
- package/dist/_chunks/agent.mjs.map +1 -1
- package/dist/_chunks/assemble.mjs +2 -0
- package/dist/_chunks/assemble.mjs.map +1 -1
- package/dist/_chunks/cache.mjs +2 -0
- package/dist/_chunks/cache.mjs.map +1 -1
- package/dist/_chunks/cache2.mjs +3 -1
- package/dist/_chunks/cache2.mjs.map +1 -1
- package/dist/_chunks/chunk.mjs +2 -0
- package/dist/_chunks/config.mjs +4 -0
- package/dist/_chunks/config.mjs.map +1 -1
- package/dist/_chunks/detect.mjs +30 -0
- package/dist/_chunks/detect.mjs.map +1 -1
- package/dist/_chunks/embedding-cache.mjs +6 -6
- package/dist/_chunks/embedding-cache.mjs.map +1 -1
- package/dist/_chunks/formatting.mjs +9 -1
- package/dist/_chunks/formatting.mjs.map +1 -1
- package/dist/_chunks/index2.d.mts +11 -2
- package/dist/_chunks/index2.d.mts.map +1 -1
- package/dist/_chunks/install.mjs +5 -3
- package/dist/_chunks/install.mjs.map +1 -1
- package/dist/_chunks/list.mjs +2 -0
- package/dist/_chunks/list.mjs.map +1 -1
- package/dist/_chunks/markdown.mjs +2 -0
- package/dist/_chunks/markdown.mjs.map +1 -1
- package/dist/_chunks/pool.mjs +2 -0
- package/dist/_chunks/pool.mjs.map +1 -1
- package/dist/_chunks/prompts.mjs +16 -0
- package/dist/_chunks/prompts.mjs.map +1 -1
- package/dist/_chunks/sanitize.mjs +3 -1
- package/dist/_chunks/sanitize.mjs.map +1 -1
- package/dist/_chunks/search-interactive.mjs +3 -1
- package/dist/_chunks/search-interactive.mjs.map +1 -1
- package/dist/_chunks/search.mjs +183 -8
- package/dist/_chunks/search.mjs.map +1 -0
- package/dist/_chunks/shared.mjs +4 -0
- package/dist/_chunks/shared.mjs.map +1 -1
- package/dist/_chunks/skills.mjs +4 -0
- package/dist/_chunks/skills.mjs.map +1 -1
- package/dist/_chunks/sources.mjs +976 -806
- package/dist/_chunks/sources.mjs.map +1 -1
- package/dist/_chunks/sync.mjs +25 -14
- package/dist/_chunks/sync.mjs.map +1 -1
- package/dist/_chunks/uninstall.mjs +3 -1
- package/dist/_chunks/uninstall.mjs.map +1 -1
- package/dist/_chunks/validate.mjs +2 -0
- package/dist/_chunks/validate.mjs.map +1 -1
- package/dist/_chunks/yaml.mjs +2 -0
- package/dist/_chunks/yaml.mjs.map +1 -1
- package/dist/agent/index.d.mts.map +1 -1
- package/dist/cli.mjs +19 -9
- package/dist/cli.mjs.map +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/retriv/index.mjs +5 -3
- package/dist/retriv/index.mjs.map +1 -1
- package/dist/retriv/worker.mjs +2 -0
- package/dist/retriv/worker.mjs.map +1 -1
- package/dist/sources/index.d.mts +2 -2
- package/dist/sources/index.mjs +2 -2
- package/dist/types.d.mts +1 -2
- package/package.json +10 -10
- package/dist/_chunks/search2.mjs +0 -180
- package/dist/_chunks/search2.mjs.map +0 -1
- package/dist/_chunks/sync2.mjs +0 -15
package/dist/_chunks/sources.mjs
CHANGED
|
@@ -14,6 +14,7 @@ import { fileURLToPath, pathToFileURL } from "node:url";
|
|
|
14
14
|
import pLimit from "p-limit";
|
|
15
15
|
import { Writable } from "node:stream";
|
|
16
16
|
import { resolvePathSync } from "mlly";
|
|
17
|
+
//#region src/sources/github-common.ts
|
|
17
18
|
/**
|
|
18
19
|
* Shared constants and helpers for GitHub source modules (issues, discussions, releases)
|
|
19
20
|
*/
|
|
@@ -33,903 +34,1053 @@ function buildFrontmatter(fields) {
|
|
|
33
34
|
lines.push("---");
|
|
34
35
|
return lines.join("\n");
|
|
35
36
|
}
|
|
37
|
+
let _ghToken;
|
|
36
38
|
/**
|
|
37
|
-
* GitHub
|
|
38
|
-
*
|
|
39
|
-
* Categorized by labels, noise filtered out, non-technical issues detected
|
|
39
|
+
* Get GitHub auth token from gh CLI (cached).
|
|
40
|
+
* Returns null if gh CLI is not available or not authenticated.
|
|
40
41
|
*/
|
|
41
|
-
|
|
42
|
+
function getGitHubToken() {
|
|
43
|
+
if (_ghToken !== void 0) return _ghToken;
|
|
44
|
+
try {
|
|
45
|
+
const { stdout } = spawnSync("gh", ["auth", "token"], {
|
|
46
|
+
encoding: "utf-8",
|
|
47
|
+
timeout: 5e3,
|
|
48
|
+
stdio: [
|
|
49
|
+
"ignore",
|
|
50
|
+
"pipe",
|
|
51
|
+
"ignore"
|
|
52
|
+
]
|
|
53
|
+
});
|
|
54
|
+
_ghToken = stdout?.trim() || null;
|
|
55
|
+
} catch {
|
|
56
|
+
_ghToken = null;
|
|
57
|
+
}
|
|
58
|
+
return _ghToken;
|
|
59
|
+
}
|
|
60
|
+
/** Repos where ungh.cc failed but gh api succeeded (likely private) */
|
|
61
|
+
const _needsAuth = /* @__PURE__ */ new Set();
|
|
62
|
+
/** Mark a repo as needing authenticated access */
|
|
63
|
+
function markRepoPrivate(owner, repo) {
|
|
64
|
+
_needsAuth.add(`${owner}/${repo}`);
|
|
65
|
+
}
|
|
66
|
+
/** Check if a repo is known to need authenticated access */
|
|
67
|
+
function isKnownPrivateRepo(owner, repo) {
|
|
68
|
+
return _needsAuth.has(`${owner}/${repo}`);
|
|
69
|
+
}
|
|
70
|
+
const GH_API = "https://api.github.com";
|
|
71
|
+
const ghApiFetch = ofetch.create({
|
|
72
|
+
retry: 2,
|
|
73
|
+
retryDelay: 500,
|
|
74
|
+
timeout: 15e3,
|
|
75
|
+
headers: { "User-Agent": "skilld/1.0" }
|
|
76
|
+
});
|
|
77
|
+
const LINK_NEXT_RE = /<([^>]+)>;\s*rel="next"/;
|
|
78
|
+
/** Parse GitHub Link header for next page URL */
|
|
79
|
+
function parseLinkNext(header) {
|
|
80
|
+
if (!header) return null;
|
|
81
|
+
return header.match(LINK_NEXT_RE)?.[1] ?? null;
|
|
82
|
+
}
|
|
42
83
|
/**
|
|
43
|
-
*
|
|
84
|
+
* Authenticated fetch against api.github.com. Returns null if no token or request fails.
|
|
85
|
+
* Endpoint should be relative, e.g. `repos/owner/repo/releases`.
|
|
44
86
|
*/
|
|
45
|
-
function
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
return
|
|
87
|
+
async function ghApi(endpoint) {
|
|
88
|
+
const token = getGitHubToken();
|
|
89
|
+
if (!token) return null;
|
|
90
|
+
return ghApiFetch(`${GH_API}/${endpoint}`, { headers: { Authorization: `token ${token}` } }).catch(() => null);
|
|
49
91
|
}
|
|
50
|
-
/** Labels that indicate noise — filter these out entirely */
|
|
51
|
-
const NOISE_LABELS = new Set([
|
|
52
|
-
"duplicate",
|
|
53
|
-
"stale",
|
|
54
|
-
"invalid",
|
|
55
|
-
"wontfix",
|
|
56
|
-
"won't fix",
|
|
57
|
-
"spam",
|
|
58
|
-
"off-topic",
|
|
59
|
-
"needs triage",
|
|
60
|
-
"triage"
|
|
61
|
-
]);
|
|
62
|
-
/** Labels that indicate feature requests — deprioritize */
|
|
63
|
-
const FEATURE_LABELS = new Set([
|
|
64
|
-
"enhancement",
|
|
65
|
-
"feature",
|
|
66
|
-
"feature request",
|
|
67
|
-
"feature-request",
|
|
68
|
-
"proposal",
|
|
69
|
-
"rfc",
|
|
70
|
-
"idea",
|
|
71
|
-
"suggestion"
|
|
72
|
-
]);
|
|
73
|
-
const BUG_LABELS = new Set([
|
|
74
|
-
"bug",
|
|
75
|
-
"defect",
|
|
76
|
-
"regression",
|
|
77
|
-
"error",
|
|
78
|
-
"crash",
|
|
79
|
-
"fix",
|
|
80
|
-
"confirmed",
|
|
81
|
-
"verified"
|
|
82
|
-
]);
|
|
83
|
-
const QUESTION_LABELS = new Set([
|
|
84
|
-
"question",
|
|
85
|
-
"help wanted",
|
|
86
|
-
"support",
|
|
87
|
-
"usage",
|
|
88
|
-
"how-to",
|
|
89
|
-
"help",
|
|
90
|
-
"assistance"
|
|
91
|
-
]);
|
|
92
|
-
const DOCS_LABELS = new Set([
|
|
93
|
-
"documentation",
|
|
94
|
-
"docs",
|
|
95
|
-
"doc",
|
|
96
|
-
"typo"
|
|
97
|
-
]);
|
|
98
92
|
/**
|
|
99
|
-
*
|
|
100
|
-
*
|
|
93
|
+
* Paginated GitHub API fetch. Follows Link headers, returns concatenated arrays.
|
|
94
|
+
* Endpoint should return a JSON array, e.g. `repos/owner/repo/releases`.
|
|
101
95
|
*/
|
|
102
|
-
function
|
|
103
|
-
|
|
104
|
-
return
|
|
96
|
+
async function ghApiPaginated(endpoint) {
|
|
97
|
+
const token = getGitHubToken();
|
|
98
|
+
if (!token) return [];
|
|
99
|
+
const headers = { Authorization: `token ${token}` };
|
|
100
|
+
const results = [];
|
|
101
|
+
let url = `${GH_API}/${endpoint}`;
|
|
102
|
+
while (url) {
|
|
103
|
+
const res = await ghApiFetch.raw(url, { headers }).catch(() => null);
|
|
104
|
+
if (!res?.ok || !Array.isArray(res._data)) break;
|
|
105
|
+
results.push(...res._data);
|
|
106
|
+
url = parseLinkNext(res.headers.get("link"));
|
|
107
|
+
}
|
|
108
|
+
return results;
|
|
105
109
|
}
|
|
110
|
+
//#endregion
|
|
111
|
+
//#region src/sources/utils.ts
|
|
106
112
|
/**
|
|
107
|
-
*
|
|
113
|
+
* Shared utilities for doc resolution
|
|
108
114
|
*/
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
return "other";
|
|
116
|
-
}
|
|
115
|
+
const $fetch = ofetch.create({
|
|
116
|
+
retry: 3,
|
|
117
|
+
retryDelay: 500,
|
|
118
|
+
timeout: 15e3,
|
|
119
|
+
headers: { "User-Agent": "skilld/1.0" }
|
|
120
|
+
});
|
|
117
121
|
/**
|
|
118
|
-
*
|
|
122
|
+
* Fetch text content from URL
|
|
119
123
|
*/
|
|
120
|
-
function
|
|
121
|
-
|
|
122
|
-
if (issue.title.startsWith("☂️") || issue.title.startsWith("[META]") || issue.title.startsWith("[Tracking]")) return true;
|
|
123
|
-
return false;
|
|
124
|
+
async function fetchText(url) {
|
|
125
|
+
return $fetch(url, { responseType: "text" }).catch(() => null);
|
|
124
126
|
}
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
127
|
+
const RAW_GH_RE = /raw\.githubusercontent\.com\/([^/]+)\/([^/]+)/;
|
|
128
|
+
/** Extract owner/repo from a GitHub raw content URL */
|
|
129
|
+
function extractGitHubRepo(url) {
|
|
130
|
+
const match = url.match(RAW_GH_RE);
|
|
131
|
+
return match ? {
|
|
132
|
+
owner: match[1],
|
|
133
|
+
repo: match[2]
|
|
134
|
+
} : null;
|
|
128
135
|
}
|
|
129
136
|
/**
|
|
130
|
-
*
|
|
131
|
-
*
|
|
132
|
-
*
|
|
137
|
+
* Fetch text from a GitHub raw URL with auth fallback for private repos.
|
|
138
|
+
* Tries unauthenticated first (fast path), falls back to authenticated
|
|
139
|
+
* request when the repo is known to be private or unauthenticated fails.
|
|
140
|
+
*
|
|
141
|
+
* Only sends auth tokens to raw.githubusercontent.com — returns null for
|
|
142
|
+
* non-GitHub URLs that fail unauthenticated to prevent token leakage.
|
|
133
143
|
*/
|
|
134
|
-
function
|
|
135
|
-
const
|
|
136
|
-
if (
|
|
137
|
-
|
|
138
|
-
|
|
144
|
+
async function fetchGitHubRaw(url) {
|
|
145
|
+
const gh = extractGitHubRepo(url);
|
|
146
|
+
if (!(gh ? isKnownPrivateRepo(gh.owner, gh.repo) : false)) {
|
|
147
|
+
const content = await fetchText(url);
|
|
148
|
+
if (content) return content;
|
|
149
|
+
}
|
|
150
|
+
if (!gh) return null;
|
|
151
|
+
const token = getGitHubToken();
|
|
152
|
+
if (!token) return null;
|
|
153
|
+
const content = await $fetch(url, {
|
|
154
|
+
responseType: "text",
|
|
155
|
+
headers: { Authorization: `token ${token}` }
|
|
156
|
+
}).catch(() => null);
|
|
157
|
+
if (content) markRepoPrivate(gh.owner, gh.repo);
|
|
158
|
+
return content;
|
|
139
159
|
}
|
|
140
160
|
/**
|
|
141
|
-
*
|
|
142
|
-
* Steep decay so recent issues dominate over old high-reaction ones.
|
|
143
|
-
* At 0.6: 1yr=0.63x, 2yr=0.45x, 4yr=0.29x, 6yr=0.22x
|
|
161
|
+
* Verify URL exists and is not HTML (likely 404 page)
|
|
144
162
|
*/
|
|
145
|
-
function
|
|
146
|
-
|
|
163
|
+
async function verifyUrl(url) {
|
|
164
|
+
const res = await $fetch.raw(url, { method: "HEAD" }).catch(() => null);
|
|
165
|
+
if (!res) return false;
|
|
166
|
+
return !(res.headers.get("content-type") || "").includes("text/html");
|
|
147
167
|
}
|
|
148
168
|
/**
|
|
149
|
-
*
|
|
150
|
-
* Bugs and questions get priority; feature requests are hard-capped.
|
|
169
|
+
* Check if URL points to a social media or package registry site (not real docs)
|
|
151
170
|
*/
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
for (let i = 0; i < take; i++) {
|
|
170
|
-
selected.push(group[i]);
|
|
171
|
-
used.add(group[i].number);
|
|
172
|
-
remaining--;
|
|
173
|
-
}
|
|
171
|
+
const USELESS_HOSTS = new Set([
|
|
172
|
+
"twitter.com",
|
|
173
|
+
"x.com",
|
|
174
|
+
"facebook.com",
|
|
175
|
+
"linkedin.com",
|
|
176
|
+
"youtube.com",
|
|
177
|
+
"instagram.com",
|
|
178
|
+
"npmjs.com",
|
|
179
|
+
"www.npmjs.com",
|
|
180
|
+
"yarnpkg.com"
|
|
181
|
+
]);
|
|
182
|
+
function isUselessDocsUrl(url) {
|
|
183
|
+
try {
|
|
184
|
+
const { hostname } = new URL(url);
|
|
185
|
+
return USELESS_HOSTS.has(hostname);
|
|
186
|
+
} catch {
|
|
187
|
+
return false;
|
|
174
188
|
}
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Check if URL is a GitHub repo URL (not a docs site)
|
|
192
|
+
*/
|
|
193
|
+
function isGitHubRepoUrl(url) {
|
|
194
|
+
try {
|
|
195
|
+
const parsed = new URL(url);
|
|
196
|
+
return parsed.hostname === "github.com" || parsed.hostname === "www.github.com";
|
|
197
|
+
} catch {
|
|
198
|
+
return false;
|
|
182
199
|
}
|
|
183
|
-
return selected.sort((a, b) => b.score - a.score);
|
|
184
200
|
}
|
|
185
201
|
/**
|
|
186
|
-
*
|
|
202
|
+
* Parse owner/repo from GitHub URL
|
|
187
203
|
*/
|
|
188
|
-
function
|
|
189
|
-
|
|
190
|
-
if (
|
|
191
|
-
return
|
|
204
|
+
function parseGitHubUrl(url) {
|
|
205
|
+
const match = url.match(/github\.com\/([^/]+)\/([^/]+?)(?:\.git)?(?:[/#]|$)/);
|
|
206
|
+
if (!match) return null;
|
|
207
|
+
return {
|
|
208
|
+
owner: match[1],
|
|
209
|
+
repo: match[2]
|
|
210
|
+
};
|
|
192
211
|
}
|
|
193
212
|
/**
|
|
194
|
-
*
|
|
195
|
-
* Instead of slicing at a char limit, finds a safe break point.
|
|
213
|
+
* Normalize git repo URL to https
|
|
196
214
|
*/
|
|
197
|
-
function
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
215
|
+
function normalizeRepoUrl(url) {
|
|
216
|
+
return url.replace(/^git\+/, "").replace(/#.*$/, "").replace(/\.git$/, "").replace(/^git:\/\//, "https://").replace(/^ssh:\/\/git@github\.com/, "https://github.com").replace(/^git@github\.com:/, "https://github.com/");
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Parse package spec with optional dist-tag or version: "vue@beta" → { name: "vue", tag: "beta" }
|
|
220
|
+
* Handles scoped packages: "@vue/reactivity@beta" → { name: "@vue/reactivity", tag: "beta" }
|
|
221
|
+
*/
|
|
222
|
+
function parsePackageSpec(spec) {
|
|
223
|
+
if (spec.startsWith("@")) {
|
|
224
|
+
const slashIdx = spec.indexOf("/");
|
|
225
|
+
if (slashIdx !== -1) {
|
|
226
|
+
const atIdx = spec.indexOf("@", slashIdx + 1);
|
|
227
|
+
if (atIdx !== -1) return {
|
|
228
|
+
name: spec.slice(0, atIdx),
|
|
229
|
+
tag: spec.slice(atIdx + 1)
|
|
230
|
+
};
|
|
209
231
|
}
|
|
232
|
+
return { name: spec };
|
|
210
233
|
}
|
|
211
|
-
const
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
234
|
+
const atIdx = spec.indexOf("@");
|
|
235
|
+
if (atIdx !== -1) return {
|
|
236
|
+
name: spec.slice(0, atIdx),
|
|
237
|
+
tag: spec.slice(atIdx + 1)
|
|
238
|
+
};
|
|
239
|
+
return { name: spec };
|
|
215
240
|
}
|
|
216
241
|
/**
|
|
217
|
-
*
|
|
242
|
+
* Extract branch hint from URL fragment (e.g. "git+https://...#main" → "main")
|
|
218
243
|
*/
|
|
219
|
-
function
|
|
220
|
-
const
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
date.setMonth(date.getMonth() + 6);
|
|
226
|
-
datePart = `+closed:<=${isoDate(date.toISOString())}`;
|
|
227
|
-
} else datePart = `+closed:>${oneYearAgo()}`;
|
|
228
|
-
else if (releasedAt) {
|
|
229
|
-
const date = new Date(releasedAt);
|
|
230
|
-
date.setMonth(date.getMonth() + 6);
|
|
231
|
-
datePart = `+created:<=${isoDate(date.toISOString())}`;
|
|
232
|
-
}
|
|
233
|
-
const { stdout: result } = spawnSync("gh", [
|
|
234
|
-
"api",
|
|
235
|
-
`search/issues?q=${`repo:${owner}/${repo}+is:issue+is:${state}${datePart}`}&sort=reactions&order=desc&per_page=${fetchCount}`,
|
|
236
|
-
"-q",
|
|
237
|
-
".items[] | {number, title, state, labels: [.labels[]?.name], body, createdAt: .created_at, url: .html_url, reactions: .reactions[\"+1\"], comments: .comments, user: .user.login, userType: .user.type, authorAssociation: .author_association}"
|
|
238
|
-
], {
|
|
239
|
-
encoding: "utf-8",
|
|
240
|
-
maxBuffer: 10 * 1024 * 1024
|
|
241
|
-
});
|
|
242
|
-
if (!result) return [];
|
|
243
|
-
return result.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)).filter((issue) => !BOT_USERS.has(issue.user) && issue.userType !== "Bot").filter((issue) => !isNoiseIssue(issue)).filter((issue) => !isNonTechnical(issue)).map(({ user: _, userType: __, authorAssociation, ...issue }) => {
|
|
244
|
-
const isMaintainer = [
|
|
245
|
-
"OWNER",
|
|
246
|
-
"MEMBER",
|
|
247
|
-
"COLLABORATOR"
|
|
248
|
-
].includes(authorAssociation);
|
|
249
|
-
const isRoadmap = /\broadmap\b/i.test(issue.title) || issue.labels.some((l) => /roadmap/i.test(l));
|
|
250
|
-
return {
|
|
251
|
-
...issue,
|
|
252
|
-
type: classifyIssue(issue.labels),
|
|
253
|
-
topComments: [],
|
|
254
|
-
score: freshnessScore(issue.reactions, issue.createdAt) * (isMaintainer && isRoadmap ? 5 : 1)
|
|
255
|
-
};
|
|
256
|
-
}).sort((a, b) => b.score - a.score).slice(0, count);
|
|
257
|
-
}
|
|
258
|
-
function oneYearAgo() {
|
|
259
|
-
const d = /* @__PURE__ */ new Date();
|
|
260
|
-
d.setFullYear(d.getFullYear() - 1);
|
|
261
|
-
return isoDate(d.toISOString());
|
|
244
|
+
function extractBranchHint(url) {
|
|
245
|
+
const hash = url.indexOf("#");
|
|
246
|
+
if (hash === -1) return void 0;
|
|
247
|
+
const fragment = url.slice(hash + 1);
|
|
248
|
+
if (!fragment || fragment === "readme") return void 0;
|
|
249
|
+
return fragment;
|
|
262
250
|
}
|
|
263
|
-
|
|
264
|
-
|
|
251
|
+
//#endregion
|
|
252
|
+
//#region src/sources/releases.ts
|
|
265
253
|
/**
|
|
266
|
-
*
|
|
267
|
-
* Enriches the top N highest-score issues with their best comments.
|
|
268
|
-
* Prioritizes: comments with code blocks, from maintainers, with high reactions.
|
|
269
|
-
* Filters out "+1", "any updates?", "same here" noise.
|
|
254
|
+
* GitHub release notes fetching via GitHub API (preferred) with ungh.cc fallback
|
|
270
255
|
*/
|
|
271
|
-
function
|
|
272
|
-
const
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
"-f",
|
|
282
|
-
`owner=${owner}`,
|
|
283
|
-
"-f",
|
|
284
|
-
`repo=${repo}`
|
|
285
|
-
], {
|
|
286
|
-
encoding: "utf-8",
|
|
287
|
-
maxBuffer: 10 * 1024 * 1024
|
|
288
|
-
});
|
|
289
|
-
if (!result) return;
|
|
290
|
-
const repo_ = JSON.parse(result)?.data?.repository;
|
|
291
|
-
if (!repo_) return;
|
|
292
|
-
for (let i = 0; i < worth.length; i++) {
|
|
293
|
-
const nodes = repo_[`i${i}`]?.comments?.nodes;
|
|
294
|
-
if (!Array.isArray(nodes)) continue;
|
|
295
|
-
const issue = worth[i];
|
|
296
|
-
const comments = nodes.filter((c) => c.author && !BOT_USERS.has(c.author.login)).filter((c) => !COMMENT_NOISE_RE$1.test((c.body || "").trim())).map((c) => {
|
|
297
|
-
const isMaintainer = [
|
|
298
|
-
"OWNER",
|
|
299
|
-
"MEMBER",
|
|
300
|
-
"COLLABORATOR"
|
|
301
|
-
].includes(c.authorAssociation);
|
|
302
|
-
const body = c.body || "";
|
|
303
|
-
const reactions = c.reactions?.totalCount || 0;
|
|
304
|
-
const _score = (isMaintainer ? 3 : 1) * (hasCodeBlock$1(body) ? 2 : 1) * (1 + reactions);
|
|
305
|
-
return {
|
|
306
|
-
body,
|
|
307
|
-
author: c.author.login,
|
|
308
|
-
reactions,
|
|
309
|
-
isMaintainer,
|
|
310
|
-
_score
|
|
311
|
-
};
|
|
312
|
-
}).sort((a, b) => b._score - a._score);
|
|
313
|
-
issue.topComments = comments.slice(0, 3).map(({ _score: _, ...c }) => c);
|
|
314
|
-
if (issue.state === "closed") issue.resolvedIn = detectResolvedVersion(comments);
|
|
315
|
-
}
|
|
316
|
-
} catch {}
|
|
256
|
+
function parseSemver(version) {
|
|
257
|
+
const clean = version.replace(/^v/, "");
|
|
258
|
+
const match = clean.match(/^(\d+)(?:\.(\d+))?(?:\.(\d+))?/);
|
|
259
|
+
if (!match) return null;
|
|
260
|
+
return {
|
|
261
|
+
major: +match[1],
|
|
262
|
+
minor: match[2] ? +match[2] : 0,
|
|
263
|
+
patch: match[3] ? +match[3] : 0,
|
|
264
|
+
raw: clean
|
|
265
|
+
};
|
|
317
266
|
}
|
|
318
267
|
/**
|
|
319
|
-
*
|
|
320
|
-
*
|
|
268
|
+
* Extract version from a release tag, handling monorepo formats:
|
|
269
|
+
* - `pkg@1.2.3` → `1.2.3`
|
|
270
|
+
* - `pkg-v1.2.3` → `1.2.3`
|
|
271
|
+
* - `v1.2.3` → `1.2.3`
|
|
272
|
+
* - `1.2.3` → `1.2.3`
|
|
321
273
|
*/
|
|
322
|
-
function
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
if (
|
|
328
|
-
const vMatch = c.body.match(/\bv?(\d+\.\d+\.\d+)\b/);
|
|
329
|
-
if (vMatch) return vMatch[1];
|
|
330
|
-
}
|
|
274
|
+
function extractVersion(tag, packageName) {
|
|
275
|
+
if (packageName) {
|
|
276
|
+
const atMatch = tag.match(new RegExp(`^${escapeRegex(packageName)}@(.+)$`));
|
|
277
|
+
if (atMatch) return atMatch[1];
|
|
278
|
+
const dashMatch = tag.match(new RegExp(`^${escapeRegex(packageName)}-v?(.+)$`));
|
|
279
|
+
if (dashMatch) return dashMatch[1];
|
|
331
280
|
}
|
|
281
|
+
return tag.replace(/^v/, "");
|
|
282
|
+
}
|
|
283
|
+
function escapeRegex(str) {
|
|
284
|
+
return str.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
332
285
|
}
|
|
333
286
|
/**
|
|
334
|
-
*
|
|
335
|
-
* Returns a balanced mix: bugs > questions > docs > other > features.
|
|
336
|
-
* Filters noise, non-technical content, and enriches with quality comments.
|
|
287
|
+
* Check if a release tag belongs to a specific package
|
|
337
288
|
*/
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
const openCount = Math.ceil(limit * .75);
|
|
341
|
-
const closedCount = limit - openCount;
|
|
342
|
-
try {
|
|
343
|
-
const open = fetchIssuesByState(owner, repo, "open", Math.min(openCount * 2, 100), releasedAt, fromDate);
|
|
344
|
-
const closed = fetchIssuesByState(owner, repo, "closed", Math.min(closedCount * 2, 50), releasedAt, fromDate);
|
|
345
|
-
const selected = applyTypeQuotas([...open, ...closed], limit);
|
|
346
|
-
enrichWithComments(owner, repo, selected);
|
|
347
|
-
return selected;
|
|
348
|
-
} catch {
|
|
349
|
-
return [];
|
|
350
|
-
}
|
|
289
|
+
function tagMatchesPackage(tag, packageName) {
|
|
290
|
+
return tag.startsWith(`${packageName}@`) || tag.startsWith(`${packageName}-v`) || tag.startsWith(`${packageName}-`);
|
|
351
291
|
}
|
|
352
292
|
/**
|
|
353
|
-
*
|
|
293
|
+
* Check if a version string contains a prerelease suffix (e.g. 6.0.0-beta, 1.2.3-rc.1)
|
|
354
294
|
*/
|
|
355
|
-
function
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
295
|
+
function isPrerelease(version) {
|
|
296
|
+
return /^\d+\.\d+\.\d+-.+/.test(version.replace(/^v/, ""));
|
|
297
|
+
}
|
|
298
|
+
function compareSemver(a, b) {
|
|
299
|
+
if (a.major !== b.major) return a.major - b.major;
|
|
300
|
+
if (a.minor !== b.minor) return a.minor - b.minor;
|
|
301
|
+
return a.patch - b.patch;
|
|
302
|
+
}
|
|
303
|
+
/** Map GitHub API release to our GitHubRelease shape */
|
|
304
|
+
function mapApiRelease(r) {
|
|
305
|
+
return {
|
|
306
|
+
id: r.id,
|
|
307
|
+
tag: r.tag_name,
|
|
308
|
+
name: r.name,
|
|
309
|
+
prerelease: r.prerelease,
|
|
310
|
+
createdAt: r.created_at,
|
|
311
|
+
publishedAt: r.published_at,
|
|
312
|
+
markdown: r.body
|
|
366
313
|
};
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
314
|
+
}
|
|
315
|
+
/**
|
|
316
|
+
* Fetch all releases — GitHub API first (authenticated, async), ungh.cc fallback
|
|
317
|
+
*/
|
|
318
|
+
async function fetchAllReleases(owner, repo) {
|
|
319
|
+
const apiReleases = await ghApiPaginated(`repos/${owner}/${repo}/releases`);
|
|
320
|
+
if (apiReleases.length > 0) return apiReleases.map(mapApiRelease);
|
|
321
|
+
return (await $fetch(`https://ungh.cc/repos/${owner}/${repo}/releases`, { signal: AbortSignal.timeout(15e3) }).catch(() => null))?.releases ?? [];
|
|
322
|
+
}
|
|
323
|
+
/**
|
|
324
|
+
* Select last 20 stable releases for a package, sorted newest first.
|
|
325
|
+
* For monorepos, filters to package-specific tags (pkg@version).
|
|
326
|
+
* Falls back to generic tags (v1.2.3) only if no package-specific found.
|
|
327
|
+
* If installedVersion is provided, filters out releases newer than it.
|
|
328
|
+
*/
|
|
329
|
+
function selectReleases(releases, packageName, installedVersion, fromDate) {
|
|
330
|
+
const hasMonorepoTags = packageName && releases.some((r) => tagMatchesPackage(r.tag, packageName));
|
|
331
|
+
const installedSv = installedVersion ? parseSemver(installedVersion) : null;
|
|
332
|
+
const installedIsPrerelease = installedVersion ? isPrerelease(installedVersion) : false;
|
|
333
|
+
const fromTs = fromDate ? new Date(fromDate).getTime() : null;
|
|
334
|
+
const sorted = releases.filter((r) => {
|
|
335
|
+
const ver = extractVersion(r.tag, hasMonorepoTags ? packageName : void 0);
|
|
336
|
+
if (!ver) return false;
|
|
337
|
+
const sv = parseSemver(ver);
|
|
338
|
+
if (!sv) return false;
|
|
339
|
+
if (hasMonorepoTags && packageName && !tagMatchesPackage(r.tag, packageName)) return false;
|
|
340
|
+
if (fromTs) {
|
|
341
|
+
const pubDate = r.publishedAt || r.createdAt;
|
|
342
|
+
if (pubDate && new Date(pubDate).getTime() < fromTs) return false;
|
|
385
343
|
}
|
|
386
|
-
|
|
387
|
-
|
|
344
|
+
if (r.prerelease) {
|
|
345
|
+
if (!installedIsPrerelease || !installedSv) return false;
|
|
346
|
+
return sv.major === installedSv.major && sv.minor === installedSv.minor;
|
|
347
|
+
}
|
|
348
|
+
if (installedSv && compareSemver(sv, installedSv) > 0) return false;
|
|
349
|
+
return true;
|
|
350
|
+
}).sort((a, b) => {
|
|
351
|
+
const verA = extractVersion(a.tag, hasMonorepoTags ? packageName : void 0);
|
|
352
|
+
const verB = extractVersion(b.tag, hasMonorepoTags ? packageName : void 0);
|
|
353
|
+
if (!verA || !verB) return 0;
|
|
354
|
+
return compareSemver(parseSemver(verB), parseSemver(verA));
|
|
355
|
+
});
|
|
356
|
+
return fromDate ? sorted : sorted.slice(0, 20);
|
|
388
357
|
}
|
|
389
358
|
/**
|
|
390
|
-
*
|
|
391
|
-
* Groups by type so the LLM can quickly find bugs vs questions.
|
|
359
|
+
* Format a release as markdown with YAML frontmatter
|
|
392
360
|
*/
|
|
393
|
-
function
|
|
394
|
-
const
|
|
395
|
-
|
|
396
|
-
const
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
other: "Other"
|
|
402
|
-
};
|
|
403
|
-
const typeOrder = [
|
|
404
|
-
"bug",
|
|
405
|
-
"question",
|
|
406
|
-
"docs",
|
|
407
|
-
"other",
|
|
408
|
-
"feature"
|
|
361
|
+
function formatRelease(release, packageName) {
|
|
362
|
+
const date = isoDate(release.publishedAt || release.createdAt);
|
|
363
|
+
const version = extractVersion(release.tag, packageName) || release.tag;
|
|
364
|
+
const fm = [
|
|
365
|
+
"---",
|
|
366
|
+
`tag: ${release.tag}`,
|
|
367
|
+
`version: ${version}`,
|
|
368
|
+
`published: ${date}`
|
|
409
369
|
];
|
|
410
|
-
|
|
370
|
+
if (release.name && release.name !== release.tag) fm.push(`name: "${release.name.replace(/"/g, "\\\"")}"`);
|
|
371
|
+
fm.push("---");
|
|
372
|
+
return `${fm.join("\n")}\n\n# ${release.name || release.tag}\n\n${release.markdown}`;
|
|
373
|
+
}
|
|
374
|
+
/**
|
|
375
|
+
* Generate a unified summary index of all releases for quick LLM scanning.
|
|
376
|
+
* Includes GitHub releases, blog release posts, and CHANGELOG link.
|
|
377
|
+
*/
|
|
378
|
+
function generateReleaseIndex(releasesOrOpts, packageName) {
|
|
379
|
+
const opts = Array.isArray(releasesOrOpts) ? {
|
|
380
|
+
releases: releasesOrOpts,
|
|
381
|
+
packageName
|
|
382
|
+
} : releasesOrOpts;
|
|
383
|
+
const { releases, blogReleases, hasChangelog } = opts;
|
|
384
|
+
const pkg = opts.packageName;
|
|
385
|
+
const lines = [
|
|
411
386
|
[
|
|
412
387
|
"---",
|
|
413
|
-
`total: ${
|
|
414
|
-
`
|
|
415
|
-
`closed: ${issues.filter((i) => i.state !== "open").length}`,
|
|
388
|
+
`total: ${releases.length + (blogReleases?.length ?? 0)}`,
|
|
389
|
+
`latest: ${releases[0]?.tag || "unknown"}`,
|
|
416
390
|
"---"
|
|
417
391
|
].join("\n"),
|
|
418
392
|
"",
|
|
419
|
-
"#
|
|
393
|
+
"# Releases Index",
|
|
420
394
|
""
|
|
421
395
|
];
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
const date = isoDate(
|
|
431
|
-
|
|
396
|
+
if (blogReleases && blogReleases.length > 0) {
|
|
397
|
+
lines.push("## Blog Releases", "");
|
|
398
|
+
for (const b of blogReleases) lines.push(`- [${b.version}](./blog-${b.version}.md): ${b.title} (${b.date})`);
|
|
399
|
+
lines.push("");
|
|
400
|
+
}
|
|
401
|
+
if (releases.length > 0) {
|
|
402
|
+
if (blogReleases && blogReleases.length > 0) lines.push("## Release Notes", "");
|
|
403
|
+
for (const r of releases) {
|
|
404
|
+
const date = isoDate(r.publishedAt || r.createdAt);
|
|
405
|
+
const filename = r.tag.includes("@") || r.tag.startsWith("v") ? r.tag : `v${r.tag}`;
|
|
406
|
+
const sv = parseSemver(extractVersion(r.tag, pkg) || r.tag);
|
|
407
|
+
const label = sv?.patch === 0 && sv.minor === 0 ? " **[MAJOR]**" : sv?.patch === 0 ? " **[MINOR]**" : "";
|
|
408
|
+
lines.push(`- [${r.tag}](./${filename}.md): ${r.name || r.tag} (${date})${label}`);
|
|
432
409
|
}
|
|
433
|
-
|
|
410
|
+
lines.push("");
|
|
434
411
|
}
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
retry: 3,
|
|
442
|
-
retryDelay: 500,
|
|
443
|
-
timeout: 15e3,
|
|
444
|
-
headers: { "User-Agent": "skilld/1.0" }
|
|
445
|
-
});
|
|
446
|
-
/**
|
|
447
|
-
* Fetch text content from URL
|
|
448
|
-
*/
|
|
449
|
-
async function fetchText(url) {
|
|
450
|
-
return $fetch(url, { responseType: "text" }).catch(() => null);
|
|
412
|
+
if (hasChangelog) {
|
|
413
|
+
lines.push("## Changelog", "");
|
|
414
|
+
lines.push("- [CHANGELOG.md](./CHANGELOG.md)");
|
|
415
|
+
lines.push("");
|
|
416
|
+
}
|
|
417
|
+
return lines.join("\n");
|
|
451
418
|
}
|
|
452
419
|
/**
|
|
453
|
-
*
|
|
420
|
+
* Check if a single release is a stub redirecting to CHANGELOG.md.
|
|
421
|
+
* Short body (<500 chars) that mentions CHANGELOG indicates no real content.
|
|
454
422
|
*/
|
|
455
|
-
|
|
456
|
-
const
|
|
457
|
-
|
|
458
|
-
return !(res.headers.get("content-type") || "").includes("text/html");
|
|
423
|
+
function isStubRelease(release) {
|
|
424
|
+
const body = (release.markdown || "").trim();
|
|
425
|
+
return body.length < 500 && /changelog\.md/i.test(body);
|
|
459
426
|
}
|
|
460
427
|
/**
|
|
461
|
-
*
|
|
428
|
+
* Fetch CHANGELOG.md from a GitHub repo at a specific ref as fallback.
|
|
429
|
+
* For monorepos, also checks packages/{shortName}/CHANGELOG.md.
|
|
462
430
|
*/
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
"npmjs.com",
|
|
471
|
-
"www.npmjs.com",
|
|
472
|
-
"yarnpkg.com"
|
|
473
|
-
]);
|
|
474
|
-
function isUselessDocsUrl(url) {
|
|
475
|
-
try {
|
|
476
|
-
const { hostname } = new URL(url);
|
|
477
|
-
return USELESS_HOSTS.has(hostname);
|
|
478
|
-
} catch {
|
|
479
|
-
return false;
|
|
431
|
+
async function fetchChangelog(owner, repo, ref, packageName) {
|
|
432
|
+
const paths = [];
|
|
433
|
+
if (packageName) {
|
|
434
|
+
const shortName = packageName.replace(/^@.*\//, "");
|
|
435
|
+
const scopeless = packageName.replace(/^@/, "").replace("/", "-");
|
|
436
|
+
const candidates = [...new Set([shortName, scopeless])];
|
|
437
|
+
for (const name of candidates) paths.push(`packages/${name}/CHANGELOG.md`);
|
|
480
438
|
}
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
function isGitHubRepoUrl(url) {
|
|
486
|
-
try {
|
|
487
|
-
const parsed = new URL(url);
|
|
488
|
-
return parsed.hostname === "github.com" || parsed.hostname === "www.github.com";
|
|
489
|
-
} catch {
|
|
490
|
-
return false;
|
|
439
|
+
paths.push("CHANGELOG.md", "changelog.md", "CHANGES.md");
|
|
440
|
+
for (const path of paths) {
|
|
441
|
+
const content = await fetchGitHubRaw(`https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${path}`);
|
|
442
|
+
if (content) return content;
|
|
491
443
|
}
|
|
444
|
+
return null;
|
|
492
445
|
}
|
|
493
446
|
/**
|
|
494
|
-
*
|
|
447
|
+
* Fetch release notes for a package. Returns CachedDoc[] with releases/{tag}.md files.
|
|
448
|
+
*
|
|
449
|
+
* Strategy:
|
|
450
|
+
* 1. Fetch GitHub releases, filter to package-specific tags for monorepos
|
|
451
|
+
* 2. If no releases found, try CHANGELOG.md as fallback
|
|
495
452
|
*/
|
|
496
|
-
function
|
|
497
|
-
const
|
|
498
|
-
if (
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
453
|
+
async function fetchReleaseNotes(owner, repo, installedVersion, gitRef, packageName, fromDate, changelogRef) {
|
|
454
|
+
const selected = selectReleases(await fetchAllReleases(owner, repo), packageName, installedVersion, fromDate);
|
|
455
|
+
if (selected.length > 0) {
|
|
456
|
+
const docs = selected.filter((r) => !isStubRelease(r)).map((r) => {
|
|
457
|
+
return {
|
|
458
|
+
path: `releases/${r.tag.includes("@") || r.tag.startsWith("v") ? r.tag : `v${r.tag}`}.md`,
|
|
459
|
+
content: formatRelease(r, packageName)
|
|
460
|
+
};
|
|
461
|
+
});
|
|
462
|
+
const changelog = await fetchChangelog(owner, repo, changelogRef || gitRef || selected[0].tag, packageName);
|
|
463
|
+
if (changelog && changelog.length < 5e5) docs.push({
|
|
464
|
+
path: "releases/CHANGELOG.md",
|
|
465
|
+
content: changelog
|
|
466
|
+
});
|
|
467
|
+
return docs;
|
|
468
|
+
}
|
|
469
|
+
const changelog = await fetchChangelog(owner, repo, changelogRef || gitRef || "main", packageName);
|
|
470
|
+
if (!changelog) return [];
|
|
471
|
+
return [{
|
|
472
|
+
path: "releases/CHANGELOG.md",
|
|
473
|
+
content: changelog
|
|
474
|
+
}];
|
|
503
475
|
}
|
|
476
|
+
//#endregion
|
|
477
|
+
//#region src/sources/blog-releases.ts
|
|
504
478
|
/**
|
|
505
|
-
*
|
|
479
|
+
* Format a blog release as markdown with YAML frontmatter
|
|
506
480
|
*/
|
|
507
|
-
function
|
|
508
|
-
return
|
|
481
|
+
function formatBlogRelease(release) {
|
|
482
|
+
return `${[
|
|
483
|
+
"---",
|
|
484
|
+
`version: ${release.version}`,
|
|
485
|
+
`title: "${release.title.replace(/"/g, "\\\"")}"`,
|
|
486
|
+
`date: ${release.date}`,
|
|
487
|
+
`url: ${release.url}`,
|
|
488
|
+
`source: blog-release`,
|
|
489
|
+
"---"
|
|
490
|
+
].join("\n")}\n\n# ${release.title}\n\n${release.markdown}`;
|
|
509
491
|
}
|
|
510
492
|
/**
|
|
511
|
-
*
|
|
512
|
-
* Handles scoped packages: "@vue/reactivity@beta" → { name: "@vue/reactivity", tag: "beta" }
|
|
493
|
+
* Fetch and parse a single blog post using preset metadata for version/date
|
|
513
494
|
*/
|
|
514
|
-
function
|
|
515
|
-
|
|
516
|
-
const
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
495
|
+
async function fetchBlogPost(entry) {
|
|
496
|
+
try {
|
|
497
|
+
const html = await $fetch(entry.url, {
|
|
498
|
+
responseType: "text",
|
|
499
|
+
signal: AbortSignal.timeout(1e4)
|
|
500
|
+
}).catch(() => null);
|
|
501
|
+
if (!html) return null;
|
|
502
|
+
let title = "";
|
|
503
|
+
const titleMatch = html.match(/<h1[^>]*>([^<]+)<\/h1>/);
|
|
504
|
+
if (titleMatch) title = titleMatch[1].trim();
|
|
505
|
+
if (!title) {
|
|
506
|
+
const metaTitleMatch = html.match(/<title>([^<]+)<\/title>/);
|
|
507
|
+
if (metaTitleMatch) title = metaTitleMatch[1].trim();
|
|
523
508
|
}
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
}
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
function extractBranchHint(url) {
|
|
537
|
-
const hash = url.indexOf("#");
|
|
538
|
-
if (hash === -1) return void 0;
|
|
539
|
-
const fragment = url.slice(hash + 1);
|
|
540
|
-
if (!fragment || fragment === "readme") return void 0;
|
|
541
|
-
return fragment;
|
|
509
|
+
const markdown = htmlToMarkdown(html);
|
|
510
|
+
if (!markdown) return null;
|
|
511
|
+
return {
|
|
512
|
+
version: entry.version,
|
|
513
|
+
title: title || entry.title || `Release ${entry.version}`,
|
|
514
|
+
date: entry.date,
|
|
515
|
+
markdown,
|
|
516
|
+
url: entry.url
|
|
517
|
+
};
|
|
518
|
+
} catch {
|
|
519
|
+
return null;
|
|
520
|
+
}
|
|
542
521
|
}
|
|
543
522
|
/**
|
|
544
|
-
*
|
|
523
|
+
* Filter blog releases by installed version
|
|
524
|
+
* Only includes releases where version <= installedVersion
|
|
525
|
+
* Returns all releases if version parsing fails (fail-safe)
|
|
545
526
|
*/
|
|
546
|
-
function
|
|
547
|
-
const
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
raw: clean
|
|
555
|
-
};
|
|
527
|
+
function filterBlogsByVersion(entries, installedVersion) {
|
|
528
|
+
const installedSv = parseSemver(installedVersion);
|
|
529
|
+
if (!installedSv) return entries;
|
|
530
|
+
return entries.filter((entry) => {
|
|
531
|
+
const entrySv = parseSemver(entry.version);
|
|
532
|
+
if (!entrySv) return false;
|
|
533
|
+
return compareSemver(entrySv, installedSv) <= 0;
|
|
534
|
+
});
|
|
556
535
|
}
|
|
557
536
|
/**
|
|
558
|
-
*
|
|
559
|
-
*
|
|
560
|
-
*
|
|
561
|
-
* - `v1.2.3` → `1.2.3`
|
|
562
|
-
* - `1.2.3` → `1.2.3`
|
|
537
|
+
* Fetch blog release notes from package presets
|
|
538
|
+
* Filters to only releases matching or older than the installed version
|
|
539
|
+
* Returns CachedDoc[] with releases/blog-{version}.md files
|
|
563
540
|
*/
|
|
564
|
-
function
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
541
|
+
async function fetchBlogReleases(packageName, installedVersion) {
|
|
542
|
+
const preset = getBlogPreset(packageName);
|
|
543
|
+
if (!preset) return [];
|
|
544
|
+
const filteredReleases = filterBlogsByVersion(preset.releases, installedVersion);
|
|
545
|
+
if (filteredReleases.length === 0) return [];
|
|
546
|
+
const releases = [];
|
|
547
|
+
const batchSize = 3;
|
|
548
|
+
for (let i = 0; i < filteredReleases.length; i += batchSize) {
|
|
549
|
+
const batch = filteredReleases.slice(i, i + batchSize);
|
|
550
|
+
const results = await Promise.all(batch.map((entry) => fetchBlogPost(entry)));
|
|
551
|
+
for (const result of results) if (result) releases.push(result);
|
|
570
552
|
}
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
553
|
+
if (releases.length === 0) return [];
|
|
554
|
+
releases.sort((a, b) => {
|
|
555
|
+
const aVer = a.version.split(".").map(Number);
|
|
556
|
+
const bVer = b.version.split(".").map(Number);
|
|
557
|
+
for (let i = 0; i < Math.max(aVer.length, bVer.length); i++) {
|
|
558
|
+
const diff = (bVer[i] ?? 0) - (aVer[i] ?? 0);
|
|
559
|
+
if (diff !== 0) return diff;
|
|
560
|
+
}
|
|
561
|
+
return 0;
|
|
562
|
+
});
|
|
563
|
+
return releases.map((r) => ({
|
|
564
|
+
path: `releases/blog-${r.version}.md`,
|
|
565
|
+
content: formatBlogRelease(r)
|
|
566
|
+
}));
|
|
575
567
|
}
|
|
568
|
+
//#endregion
|
|
569
|
+
//#region src/sources/crawl.ts
|
|
576
570
|
/**
|
|
577
|
-
*
|
|
571
|
+
* Website crawl doc source — fetches docs by crawling a URL pattern
|
|
578
572
|
*/
|
|
579
|
-
function tagMatchesPackage(tag, packageName) {
|
|
580
|
-
return tag.startsWith(`${packageName}@`) || tag.startsWith(`${packageName}-v`) || tag.startsWith(`${packageName}-`);
|
|
581
|
-
}
|
|
582
573
|
/**
|
|
583
|
-
*
|
|
574
|
+
* Crawl a URL pattern and return docs as cached doc format.
|
|
575
|
+
* Uses HTTP crawler (no browser needed) with sitemap discovery + glob filtering.
|
|
576
|
+
*
|
|
577
|
+
* @param url - URL with optional glob pattern (e.g. 'https://example.com/docs/**')
|
|
578
|
+
* @param onProgress - Optional progress callback
|
|
579
|
+
* @param maxPages - Max pages to crawl (default 200)
|
|
584
580
|
*/
|
|
585
|
-
function
|
|
586
|
-
|
|
581
|
+
async function fetchCrawledDocs(url, onProgress, maxPages = 200) {
|
|
582
|
+
const outputDir = join(tmpdir(), "skilld-crawl", Date.now().toString());
|
|
583
|
+
onProgress?.(`Crawling ${url}`);
|
|
584
|
+
const userLang = getUserLang();
|
|
585
|
+
const foreignUrls = /* @__PURE__ */ new Set();
|
|
586
|
+
const doCrawl = () => crawlAndGenerate({
|
|
587
|
+
urls: [url],
|
|
588
|
+
outputDir,
|
|
589
|
+
driver: "http",
|
|
590
|
+
generateLlmsTxt: false,
|
|
591
|
+
generateIndividualMd: true,
|
|
592
|
+
maxRequestsPerCrawl: maxPages,
|
|
593
|
+
onPage: (page) => {
|
|
594
|
+
const lang = extractHtmlLang(page.html);
|
|
595
|
+
if (lang && !lang.startsWith("en") && !lang.startsWith(userLang)) foreignUrls.add(page.url);
|
|
596
|
+
}
|
|
597
|
+
}, (progress) => {
|
|
598
|
+
if (progress.crawling.status === "processing" && progress.crawling.total > 0) onProgress?.(`Crawling ${progress.crawling.processed}/${progress.crawling.total} pages`);
|
|
599
|
+
});
|
|
600
|
+
let results = await doCrawl().catch((err) => {
|
|
601
|
+
onProgress?.(`Crawl failed: ${err?.message || err}`);
|
|
602
|
+
return [];
|
|
603
|
+
});
|
|
604
|
+
if (results.length === 0) {
|
|
605
|
+
onProgress?.("Retrying crawl");
|
|
606
|
+
results = await doCrawl().catch(() => []);
|
|
607
|
+
}
|
|
608
|
+
rmSync(outputDir, {
|
|
609
|
+
recursive: true,
|
|
610
|
+
force: true
|
|
611
|
+
});
|
|
612
|
+
const docs = [];
|
|
613
|
+
let localeFiltered = 0;
|
|
614
|
+
for (const result of results) {
|
|
615
|
+
if (!result.success || !result.content) continue;
|
|
616
|
+
if (foreignUrls.has(result.url)) {
|
|
617
|
+
localeFiltered++;
|
|
618
|
+
continue;
|
|
619
|
+
}
|
|
620
|
+
const segments = (new URL(result.url).pathname.replace(/\/$/, "") || "/index").split("/").filter(Boolean);
|
|
621
|
+
if (isForeignPathPrefix(segments[0], userLang)) {
|
|
622
|
+
localeFiltered++;
|
|
623
|
+
continue;
|
|
624
|
+
}
|
|
625
|
+
const path = `docs/${segments.join("/")}.md`;
|
|
626
|
+
docs.push({
|
|
627
|
+
path,
|
|
628
|
+
content: result.content
|
|
629
|
+
});
|
|
630
|
+
}
|
|
631
|
+
if (localeFiltered > 0) onProgress?.(`Filtered ${localeFiltered} foreign locale pages`);
|
|
632
|
+
onProgress?.(`Crawled ${docs.length} pages`);
|
|
633
|
+
return docs;
|
|
587
634
|
}
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
return
|
|
635
|
+
const HTML_LANG_RE = /<html[^>]*\slang=["']([^"']+)["']/i;
|
|
636
|
+
/** Extract lang attribute from <html> tag */
|
|
637
|
+
function extractHtmlLang(html) {
|
|
638
|
+
return HTML_LANG_RE.exec(html)?.[1]?.toLowerCase();
|
|
639
|
+
}
|
|
640
|
+
/** Common ISO 639-1 locale codes for i18n'd doc sites */
|
|
641
|
+
const LOCALE_CODES = new Set([
|
|
642
|
+
"ar",
|
|
643
|
+
"de",
|
|
644
|
+
"es",
|
|
645
|
+
"fr",
|
|
646
|
+
"id",
|
|
647
|
+
"it",
|
|
648
|
+
"ja",
|
|
649
|
+
"ko",
|
|
650
|
+
"nl",
|
|
651
|
+
"pl",
|
|
652
|
+
"pt",
|
|
653
|
+
"pt-br",
|
|
654
|
+
"ru",
|
|
655
|
+
"th",
|
|
656
|
+
"tr",
|
|
657
|
+
"uk",
|
|
658
|
+
"vi",
|
|
659
|
+
"zh",
|
|
660
|
+
"zh-cn",
|
|
661
|
+
"zh-tw"
|
|
662
|
+
]);
|
|
663
|
+
/** Check if a URL path segment is a known locale prefix foreign to both English and user's locale */
|
|
664
|
+
function isForeignPathPrefix(segment, userLang) {
|
|
665
|
+
if (!segment) return false;
|
|
666
|
+
const lower = segment.toLowerCase();
|
|
667
|
+
if (lower === "en" || lower.startsWith(userLang)) return false;
|
|
668
|
+
return LOCALE_CODES.has(lower);
|
|
669
|
+
}
|
|
670
|
+
/** Detect user's 2-letter language code from env (e.g. 'ja' from LANG=ja_JP.UTF-8) */
|
|
671
|
+
function getUserLang() {
|
|
672
|
+
const code = (process.env.LC_ALL || process.env.LANG || process.env.LANGUAGE || "").split(/[_.:-]/)[0]?.toLowerCase() || "";
|
|
673
|
+
return code.length >= 2 ? code.slice(0, 2) : "en";
|
|
674
|
+
}
|
|
675
|
+
/** Append glob pattern to a docs URL for crawling */
|
|
676
|
+
function toCrawlPattern(docsUrl) {
|
|
677
|
+
return `${docsUrl.replace(/\/+$/, "")}/**`;
|
|
592
678
|
}
|
|
679
|
+
//#endregion
|
|
680
|
+
//#region src/sources/issues.ts
|
|
681
|
+
/**
|
|
682
|
+
* GitHub issues fetching via gh CLI Search API
|
|
683
|
+
* Freshness-weighted scoring, type quotas, comment quality filtering
|
|
684
|
+
* Categorized by labels, noise filtered out, non-technical issues detected
|
|
685
|
+
*/
|
|
686
|
+
let _ghAvailable;
|
|
593
687
|
/**
|
|
594
|
-
*
|
|
688
|
+
* Check if gh CLI is installed and authenticated (cached)
|
|
595
689
|
*/
|
|
596
|
-
function
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
`repos/${owner}/${repo}/releases`,
|
|
601
|
-
"--paginate",
|
|
602
|
-
"--jq",
|
|
603
|
-
".[] | {id: .id, tag: .tag_name, name: .name, prerelease: .prerelease, createdAt: .created_at, publishedAt: .published_at, markdown: .body}"
|
|
604
|
-
], {
|
|
605
|
-
encoding: "utf-8",
|
|
606
|
-
timeout: 3e4,
|
|
607
|
-
stdio: [
|
|
608
|
-
"ignore",
|
|
609
|
-
"pipe",
|
|
610
|
-
"ignore"
|
|
611
|
-
]
|
|
612
|
-
});
|
|
613
|
-
if (!ndjson) return [];
|
|
614
|
-
return ndjson.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line));
|
|
615
|
-
} catch {
|
|
616
|
-
return [];
|
|
617
|
-
}
|
|
690
|
+
function isGhAvailable() {
|
|
691
|
+
if (_ghAvailable !== void 0) return _ghAvailable;
|
|
692
|
+
const { status } = spawnSync("gh", ["auth", "status"], { stdio: "ignore" });
|
|
693
|
+
return _ghAvailable = status === 0;
|
|
618
694
|
}
|
|
695
|
+
/** Labels that indicate noise — filter these out entirely */
|
|
696
|
+
const NOISE_LABELS = new Set([
|
|
697
|
+
"duplicate",
|
|
698
|
+
"stale",
|
|
699
|
+
"invalid",
|
|
700
|
+
"wontfix",
|
|
701
|
+
"won't fix",
|
|
702
|
+
"spam",
|
|
703
|
+
"off-topic",
|
|
704
|
+
"needs triage",
|
|
705
|
+
"triage"
|
|
706
|
+
]);
|
|
707
|
+
/** Labels that indicate feature requests — deprioritize */
|
|
708
|
+
const FEATURE_LABELS = new Set([
|
|
709
|
+
"enhancement",
|
|
710
|
+
"feature",
|
|
711
|
+
"feature request",
|
|
712
|
+
"feature-request",
|
|
713
|
+
"proposal",
|
|
714
|
+
"rfc",
|
|
715
|
+
"idea",
|
|
716
|
+
"suggestion"
|
|
717
|
+
]);
|
|
718
|
+
const BUG_LABELS = new Set([
|
|
719
|
+
"bug",
|
|
720
|
+
"defect",
|
|
721
|
+
"regression",
|
|
722
|
+
"error",
|
|
723
|
+
"crash",
|
|
724
|
+
"fix",
|
|
725
|
+
"confirmed",
|
|
726
|
+
"verified"
|
|
727
|
+
]);
|
|
728
|
+
const QUESTION_LABELS = new Set([
|
|
729
|
+
"question",
|
|
730
|
+
"help wanted",
|
|
731
|
+
"support",
|
|
732
|
+
"usage",
|
|
733
|
+
"how-to",
|
|
734
|
+
"help",
|
|
735
|
+
"assistance"
|
|
736
|
+
]);
|
|
737
|
+
const DOCS_LABELS = new Set([
|
|
738
|
+
"documentation",
|
|
739
|
+
"docs",
|
|
740
|
+
"doc",
|
|
741
|
+
"typo"
|
|
742
|
+
]);
|
|
619
743
|
/**
|
|
620
|
-
*
|
|
744
|
+
* Check if a label contains any keyword from a set.
|
|
745
|
+
* Handles emoji-prefixed labels like ":sparkles: feature request" or ":lady_beetle: bug".
|
|
621
746
|
*/
|
|
622
|
-
|
|
623
|
-
|
|
747
|
+
function labelMatchesAny(label, keywords) {
|
|
748
|
+
for (const keyword of keywords) if (label === keyword || label.includes(keyword)) return true;
|
|
749
|
+
return false;
|
|
624
750
|
}
|
|
625
751
|
/**
|
|
626
|
-
*
|
|
752
|
+
* Classify an issue by its labels into a type useful for skill generation
|
|
627
753
|
*/
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
754
|
+
function classifyIssue(labels) {
|
|
755
|
+
const lower = labels.map((l) => l.toLowerCase());
|
|
756
|
+
if (lower.some((l) => labelMatchesAny(l, BUG_LABELS))) return "bug";
|
|
757
|
+
if (lower.some((l) => labelMatchesAny(l, QUESTION_LABELS))) return "question";
|
|
758
|
+
if (lower.some((l) => labelMatchesAny(l, DOCS_LABELS))) return "docs";
|
|
759
|
+
if (lower.some((l) => labelMatchesAny(l, FEATURE_LABELS))) return "feature";
|
|
760
|
+
return "other";
|
|
634
761
|
}
|
|
635
762
|
/**
|
|
636
|
-
*
|
|
637
|
-
* For monorepos, filters to package-specific tags (pkg@version).
|
|
638
|
-
* Falls back to generic tags (v1.2.3) only if no package-specific found.
|
|
639
|
-
* If installedVersion is provided, filters out releases newer than it.
|
|
763
|
+
* Check if an issue should be filtered out entirely
|
|
640
764
|
*/
|
|
641
|
-
function
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
if (installedSv && compareSemver(sv, installedSv) > 0) return false;
|
|
661
|
-
return true;
|
|
662
|
-
}).sort((a, b) => {
|
|
663
|
-
const verA = extractVersion(a.tag, hasMonorepoTags ? packageName : void 0);
|
|
664
|
-
const verB = extractVersion(b.tag, hasMonorepoTags ? packageName : void 0);
|
|
665
|
-
if (!verA || !verB) return 0;
|
|
666
|
-
return compareSemver(parseSemver(verB), parseSemver(verA));
|
|
667
|
-
});
|
|
668
|
-
return fromDate ? sorted : sorted.slice(0, 20);
|
|
765
|
+
function isNoiseIssue(issue) {
|
|
766
|
+
if (issue.labels.map((l) => l.toLowerCase()).some((l) => labelMatchesAny(l, NOISE_LABELS))) return true;
|
|
767
|
+
if (issue.title.startsWith("☂️") || issue.title.startsWith("[META]") || issue.title.startsWith("[Tracking]")) return true;
|
|
768
|
+
return false;
|
|
769
|
+
}
|
|
770
|
+
/** Check if body contains a code block */
|
|
771
|
+
function hasCodeBlock$1(text) {
|
|
772
|
+
return /```[\s\S]*?```/.test(text) || /`[^`]+`/.test(text);
|
|
773
|
+
}
|
|
774
|
+
/**
|
|
775
|
+
* Detect non-technical issues: fan mail, showcases, sentiment.
|
|
776
|
+
* Short body + no code + high reactions = likely non-technical.
|
|
777
|
+
* Note: roadmap/tracking issues are NOT filtered — they get score-boosted instead.
|
|
778
|
+
*/
|
|
779
|
+
function isNonTechnical(issue) {
|
|
780
|
+
const body = (issue.body || "").trim();
|
|
781
|
+
if (body.length < 200 && !hasCodeBlock$1(body) && issue.reactions > 50) return true;
|
|
782
|
+
if (/\b(?:love|thank|awesome|great work)\b/i.test(issue.title) && !hasCodeBlock$1(body)) return true;
|
|
783
|
+
return false;
|
|
669
784
|
}
|
|
670
785
|
/**
|
|
671
|
-
*
|
|
786
|
+
* Freshness-weighted score: reactions * decay(age_in_years)
|
|
787
|
+
* Steep decay so recent issues dominate over old high-reaction ones.
|
|
788
|
+
* At 0.6: 1yr=0.63x, 2yr=0.45x, 4yr=0.29x, 6yr=0.22x
|
|
672
789
|
*/
|
|
673
|
-
function
|
|
674
|
-
|
|
675
|
-
const version = extractVersion(release.tag, packageName) || release.tag;
|
|
676
|
-
const fm = [
|
|
677
|
-
"---",
|
|
678
|
-
`tag: ${release.tag}`,
|
|
679
|
-
`version: ${version}`,
|
|
680
|
-
`published: ${date}`
|
|
681
|
-
];
|
|
682
|
-
if (release.name && release.name !== release.tag) fm.push(`name: "${release.name.replace(/"/g, "\\\"")}"`);
|
|
683
|
-
fm.push("---");
|
|
684
|
-
return `${fm.join("\n")}\n\n# ${release.name || release.tag}\n\n${release.markdown}`;
|
|
790
|
+
function freshnessScore(reactions, createdAt) {
|
|
791
|
+
return reactions * (1 / (1 + (Date.now() - new Date(createdAt).getTime()) / (365.25 * 24 * 60 * 60 * 1e3) * .6));
|
|
685
792
|
}
|
|
686
793
|
/**
|
|
687
|
-
*
|
|
688
|
-
*
|
|
794
|
+
* Type quotas — guarantee a mix of issue types.
|
|
795
|
+
* Bugs and questions get priority; feature requests are hard-capped.
|
|
689
796
|
*/
|
|
690
|
-
function
|
|
691
|
-
const
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
[
|
|
699
|
-
|
|
700
|
-
`total: ${releases.length + (blogReleases?.length ?? 0)}`,
|
|
701
|
-
`latest: ${releases[0]?.tag || "unknown"}`,
|
|
702
|
-
"---"
|
|
703
|
-
].join("\n"),
|
|
704
|
-
"",
|
|
705
|
-
"# Releases Index",
|
|
706
|
-
""
|
|
797
|
+
function applyTypeQuotas(issues, limit) {
|
|
798
|
+
const byType = /* @__PURE__ */ new Map();
|
|
799
|
+
for (const issue of issues) mapInsert(byType, issue.type, () => []).push(issue);
|
|
800
|
+
for (const group of byType.values()) group.sort((a, b) => b.score - a.score);
|
|
801
|
+
const quotas = [
|
|
802
|
+
["bug", Math.ceil(limit * .4)],
|
|
803
|
+
["question", Math.ceil(limit * .3)],
|
|
804
|
+
["docs", Math.ceil(limit * .15)],
|
|
805
|
+
["feature", Math.ceil(limit * .1)],
|
|
806
|
+
["other", Math.ceil(limit * .05)]
|
|
707
807
|
];
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
const sv = parseSemver(extractVersion(r.tag, pkg) || r.tag);
|
|
719
|
-
const label = sv?.patch === 0 && sv.minor === 0 ? " **[MAJOR]**" : sv?.patch === 0 ? " **[MINOR]**" : "";
|
|
720
|
-
lines.push(`- [${r.tag}](./${filename}.md): ${r.name || r.tag} (${date})${label}`);
|
|
808
|
+
const selected = [];
|
|
809
|
+
const used = /* @__PURE__ */ new Set();
|
|
810
|
+
let remaining = limit;
|
|
811
|
+
for (const [type, quota] of quotas) {
|
|
812
|
+
const group = byType.get(type) || [];
|
|
813
|
+
const take = Math.min(quota, group.length, remaining);
|
|
814
|
+
for (let i = 0; i < take; i++) {
|
|
815
|
+
selected.push(group[i]);
|
|
816
|
+
used.add(group[i].number);
|
|
817
|
+
remaining--;
|
|
721
818
|
}
|
|
722
|
-
lines.push("");
|
|
723
819
|
}
|
|
724
|
-
if (
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
820
|
+
if (remaining > 0) {
|
|
821
|
+
const unused = issues.filter((i) => !used.has(i.number) && i.type !== "feature").sort((a, b) => b.score - a.score);
|
|
822
|
+
for (const issue of unused) {
|
|
823
|
+
if (remaining <= 0) break;
|
|
824
|
+
selected.push(issue);
|
|
825
|
+
remaining--;
|
|
826
|
+
}
|
|
728
827
|
}
|
|
729
|
-
return
|
|
828
|
+
return selected.sort((a, b) => b.score - a.score);
|
|
730
829
|
}
|
|
731
830
|
/**
|
|
732
|
-
*
|
|
733
|
-
* Short body (<500 chars) that mentions CHANGELOG indicates no real content.
|
|
831
|
+
* Body truncation limit based on reactions — high-reaction issues deserve more space
|
|
734
832
|
*/
|
|
735
|
-
function
|
|
736
|
-
|
|
737
|
-
|
|
833
|
+
function bodyLimit(reactions) {
|
|
834
|
+
if (reactions >= 10) return 2e3;
|
|
835
|
+
if (reactions >= 5) return 1500;
|
|
836
|
+
return 800;
|
|
738
837
|
}
|
|
739
838
|
/**
|
|
740
|
-
*
|
|
741
|
-
*
|
|
839
|
+
* Smart body truncation — preserves code blocks and error messages.
|
|
840
|
+
* Instead of slicing at a char limit, finds a safe break point.
|
|
742
841
|
*/
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
842
|
+
function truncateBody$1(body, limit) {
|
|
843
|
+
if (body.length <= limit) return body;
|
|
844
|
+
const codeBlockRe = /```[\s\S]*?```/g;
|
|
845
|
+
let lastSafeEnd = limit;
|
|
846
|
+
let match;
|
|
847
|
+
while ((match = codeBlockRe.exec(body)) !== null) {
|
|
848
|
+
const blockStart = match.index;
|
|
849
|
+
const blockEnd = blockStart + match[0].length;
|
|
850
|
+
if (blockStart < limit && blockEnd > limit) {
|
|
851
|
+
if (blockEnd <= limit + 500) lastSafeEnd = blockEnd;
|
|
852
|
+
else lastSafeEnd = blockStart;
|
|
853
|
+
break;
|
|
854
|
+
}
|
|
750
855
|
}
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
856
|
+
const slice = body.slice(0, lastSafeEnd);
|
|
857
|
+
const lastParagraph = slice.lastIndexOf("\n\n");
|
|
858
|
+
if (lastParagraph > lastSafeEnd * .6) return `${slice.slice(0, lastParagraph)}\n\n...`;
|
|
859
|
+
return `${slice}...`;
|
|
860
|
+
}
|
|
861
|
+
/**
|
|
862
|
+
* Fetch issues for a state using GitHub Search API sorted by reactions
|
|
863
|
+
*/
|
|
864
|
+
function fetchIssuesByState(owner, repo, state, count, releasedAt, fromDate) {
|
|
865
|
+
const fetchCount = Math.min(count * 3, 100);
|
|
866
|
+
let datePart = "";
|
|
867
|
+
if (fromDate) datePart = state === "closed" ? `+closed:>=${fromDate}` : `+created:>=${fromDate}`;
|
|
868
|
+
else if (state === "closed") if (releasedAt) {
|
|
869
|
+
const date = new Date(releasedAt);
|
|
870
|
+
date.setMonth(date.getMonth() + 6);
|
|
871
|
+
datePart = `+closed:<=${isoDate(date.toISOString())}`;
|
|
872
|
+
} else datePart = `+closed:>${oneYearAgo()}`;
|
|
873
|
+
else if (releasedAt) {
|
|
874
|
+
const date = new Date(releasedAt);
|
|
875
|
+
date.setMonth(date.getMonth() + 6);
|
|
876
|
+
datePart = `+created:<=${isoDate(date.toISOString())}`;
|
|
758
877
|
}
|
|
759
|
-
|
|
878
|
+
const { stdout: result } = spawnSync("gh", [
|
|
879
|
+
"api",
|
|
880
|
+
`search/issues?q=${`repo:${owner}/${repo}+is:issue+is:${state}${datePart}`}&sort=reactions&order=desc&per_page=${fetchCount}`,
|
|
881
|
+
"-q",
|
|
882
|
+
".items[] | {number, title, state, labels: [.labels[]?.name], body, createdAt: .created_at, url: .html_url, reactions: .reactions[\"+1\"], comments: .comments, user: .user.login, userType: .user.type, authorAssociation: .author_association}"
|
|
883
|
+
], {
|
|
884
|
+
encoding: "utf-8",
|
|
885
|
+
maxBuffer: 10 * 1024 * 1024
|
|
886
|
+
});
|
|
887
|
+
if (!result) return [];
|
|
888
|
+
return result.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line)).filter((issue) => !BOT_USERS.has(issue.user) && issue.userType !== "Bot").filter((issue) => !isNoiseIssue(issue)).filter((issue) => !isNonTechnical(issue)).map(({ user: _, userType: __, authorAssociation, ...issue }) => {
|
|
889
|
+
const isMaintainer = [
|
|
890
|
+
"OWNER",
|
|
891
|
+
"MEMBER",
|
|
892
|
+
"COLLABORATOR"
|
|
893
|
+
].includes(authorAssociation);
|
|
894
|
+
const isRoadmap = /\broadmap\b/i.test(issue.title) || issue.labels.some((l) => /roadmap/i.test(l));
|
|
895
|
+
return {
|
|
896
|
+
...issue,
|
|
897
|
+
type: classifyIssue(issue.labels),
|
|
898
|
+
topComments: [],
|
|
899
|
+
score: freshnessScore(issue.reactions, issue.createdAt) * (isMaintainer && isRoadmap ? 5 : 1)
|
|
900
|
+
};
|
|
901
|
+
}).sort((a, b) => b.score - a.score).slice(0, count);
|
|
902
|
+
}
|
|
903
|
+
function oneYearAgo() {
|
|
904
|
+
const d = /* @__PURE__ */ new Date();
|
|
905
|
+
d.setFullYear(d.getFullYear() - 1);
|
|
906
|
+
return isoDate(d.toISOString());
|
|
760
907
|
}
|
|
908
|
+
/** Noise patterns in comments — filter these out */
|
|
909
|
+
const COMMENT_NOISE_RE$1 = /^(?:\+1|👍|same here|any update|bump|following|is there any progress|when will this|me too|i have the same|same issue)[\s!?.]*$/i;
|
|
761
910
|
/**
|
|
762
|
-
*
|
|
763
|
-
*
|
|
764
|
-
*
|
|
765
|
-
*
|
|
766
|
-
* 2. If no releases found, try CHANGELOG.md as fallback
|
|
911
|
+
* Batch-fetch top comments for issues via GraphQL.
|
|
912
|
+
* Enriches the top N highest-score issues with their best comments.
|
|
913
|
+
* Prioritizes: comments with code blocks, from maintainers, with high reactions.
|
|
914
|
+
* Filters out "+1", "any updates?", "same here" noise.
|
|
767
915
|
*/
|
|
768
|
-
|
|
769
|
-
const
|
|
770
|
-
if (
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
916
|
+
function enrichWithComments(owner, repo, issues, topN = 15) {
|
|
917
|
+
const worth = issues.filter((i) => i.comments > 0 && (i.type === "bug" || i.type === "question" || i.reactions >= 3)).sort((a, b) => b.score - a.score).slice(0, topN);
|
|
918
|
+
if (worth.length === 0) return;
|
|
919
|
+
const query = `query($owner: String!, $repo: String!) { repository(owner: $owner, name: $repo) { ${worth.map((issue, i) => `i${i}: issue(number: ${issue.number}) { comments(first: 10) { nodes { body author { login } authorAssociation reactions { totalCount } } } }`).join(" ")} } }`;
|
|
920
|
+
try {
|
|
921
|
+
const { stdout: result } = spawnSync("gh", [
|
|
922
|
+
"api",
|
|
923
|
+
"graphql",
|
|
924
|
+
"-f",
|
|
925
|
+
`query=${query}`,
|
|
926
|
+
"-f",
|
|
927
|
+
`owner=${owner}`,
|
|
928
|
+
"-f",
|
|
929
|
+
`repo=${repo}`
|
|
930
|
+
], {
|
|
931
|
+
encoding: "utf-8",
|
|
932
|
+
maxBuffer: 10 * 1024 * 1024
|
|
781
933
|
});
|
|
782
|
-
return
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
934
|
+
if (!result) return;
|
|
935
|
+
const repo_ = JSON.parse(result)?.data?.repository;
|
|
936
|
+
if (!repo_) return;
|
|
937
|
+
for (let i = 0; i < worth.length; i++) {
|
|
938
|
+
const nodes = repo_[`i${i}`]?.comments?.nodes;
|
|
939
|
+
if (!Array.isArray(nodes)) continue;
|
|
940
|
+
const issue = worth[i];
|
|
941
|
+
const comments = nodes.filter((c) => c.author && !BOT_USERS.has(c.author.login)).filter((c) => !COMMENT_NOISE_RE$1.test((c.body || "").trim())).map((c) => {
|
|
942
|
+
const isMaintainer = [
|
|
943
|
+
"OWNER",
|
|
944
|
+
"MEMBER",
|
|
945
|
+
"COLLABORATOR"
|
|
946
|
+
].includes(c.authorAssociation);
|
|
947
|
+
const body = c.body || "";
|
|
948
|
+
const reactions = c.reactions?.totalCount || 0;
|
|
949
|
+
const _score = (isMaintainer ? 3 : 1) * (hasCodeBlock$1(body) ? 2 : 1) * (1 + reactions);
|
|
950
|
+
return {
|
|
951
|
+
body,
|
|
952
|
+
author: c.author.login,
|
|
953
|
+
reactions,
|
|
954
|
+
isMaintainer,
|
|
955
|
+
_score
|
|
956
|
+
};
|
|
957
|
+
}).sort((a, b) => b._score - a._score);
|
|
958
|
+
issue.topComments = comments.slice(0, 3).map(({ _score: _, ...c }) => c);
|
|
959
|
+
if (issue.state === "closed") issue.resolvedIn = detectResolvedVersion(comments);
|
|
960
|
+
}
|
|
961
|
+
} catch {}
|
|
790
962
|
}
|
|
791
963
|
/**
|
|
792
|
-
*
|
|
964
|
+
* Try to detect which version fixed a closed issue from maintainer comments.
|
|
965
|
+
* Looks for version patterns in maintainer/collaborator comments.
|
|
793
966
|
*/
|
|
794
|
-
function
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
967
|
+
function detectResolvedVersion(comments) {
|
|
968
|
+
const maintainerComments = comments.filter((c) => c.isMaintainer);
|
|
969
|
+
for (const c of maintainerComments.reverse()) {
|
|
970
|
+
const match = c.body.match(/(?:fixed|landed|released|available|shipped|resolved|included)\s+in\s+v?(\d+\.\d+(?:\.\d+)?)/i);
|
|
971
|
+
if (match) return match[1];
|
|
972
|
+
if (c.body.length < 100) {
|
|
973
|
+
const vMatch = c.body.match(/\bv?(\d+\.\d+\.\d+)\b/);
|
|
974
|
+
if (vMatch) return vMatch[1];
|
|
975
|
+
}
|
|
976
|
+
}
|
|
804
977
|
}
|
|
805
978
|
/**
|
|
806
|
-
* Fetch
|
|
979
|
+
* Fetch issues from a GitHub repo with freshness-weighted scoring and type quotas.
|
|
980
|
+
* Returns a balanced mix: bugs > questions > docs > other > features.
|
|
981
|
+
* Filters noise, non-technical content, and enriches with quality comments.
|
|
807
982
|
*/
|
|
808
|
-
async function
|
|
983
|
+
async function fetchGitHubIssues(owner, repo, limit = 30, releasedAt, fromDate) {
|
|
984
|
+
if (!isGhAvailable()) return [];
|
|
985
|
+
const openCount = Math.ceil(limit * .75);
|
|
986
|
+
const closedCount = limit - openCount;
|
|
809
987
|
try {
|
|
810
|
-
const
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
let title = "";
|
|
816
|
-
const titleMatch = html.match(/<h1[^>]*>([^<]+)<\/h1>/);
|
|
817
|
-
if (titleMatch) title = titleMatch[1].trim();
|
|
818
|
-
if (!title) {
|
|
819
|
-
const metaTitleMatch = html.match(/<title>([^<]+)<\/title>/);
|
|
820
|
-
if (metaTitleMatch) title = metaTitleMatch[1].trim();
|
|
821
|
-
}
|
|
822
|
-
const markdown = htmlToMarkdown(html);
|
|
823
|
-
if (!markdown) return null;
|
|
824
|
-
return {
|
|
825
|
-
version: entry.version,
|
|
826
|
-
title: title || entry.title || `Release ${entry.version}`,
|
|
827
|
-
date: entry.date,
|
|
828
|
-
markdown,
|
|
829
|
-
url: entry.url
|
|
830
|
-
};
|
|
988
|
+
const open = fetchIssuesByState(owner, repo, "open", Math.min(openCount * 2, 100), releasedAt, fromDate);
|
|
989
|
+
const closed = fetchIssuesByState(owner, repo, "closed", Math.min(closedCount * 2, 50), releasedAt, fromDate);
|
|
990
|
+
const selected = applyTypeQuotas([...open, ...closed], limit);
|
|
991
|
+
enrichWithComments(owner, repo, selected);
|
|
992
|
+
return selected;
|
|
831
993
|
} catch {
|
|
832
|
-
return
|
|
994
|
+
return [];
|
|
833
995
|
}
|
|
834
996
|
}
|
|
835
997
|
/**
|
|
836
|
-
*
|
|
837
|
-
* Only includes releases where version <= installedVersion
|
|
838
|
-
* Returns all releases if version parsing fails (fail-safe)
|
|
839
|
-
*/
|
|
840
|
-
function filterBlogsByVersion(entries, installedVersion) {
|
|
841
|
-
const installedSv = parseSemver(installedVersion);
|
|
842
|
-
if (!installedSv) return entries;
|
|
843
|
-
return entries.filter((entry) => {
|
|
844
|
-
const entrySv = parseSemver(entry.version);
|
|
845
|
-
if (!entrySv) return false;
|
|
846
|
-
return compareSemver(entrySv, installedSv) <= 0;
|
|
847
|
-
});
|
|
848
|
-
}
|
|
849
|
-
/**
|
|
850
|
-
* Fetch blog release notes from package presets
|
|
851
|
-
* Filters to only releases matching or older than the installed version
|
|
852
|
-
* Returns CachedDoc[] with releases/blog-{version}.md files
|
|
998
|
+
* Format a single issue as markdown with YAML frontmatter
|
|
853
999
|
*/
|
|
854
|
-
|
|
855
|
-
const
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
1000
|
+
function formatIssueAsMarkdown(issue) {
|
|
1001
|
+
const limit = bodyLimit(issue.reactions);
|
|
1002
|
+
const fmFields = {
|
|
1003
|
+
number: issue.number,
|
|
1004
|
+
title: issue.title,
|
|
1005
|
+
type: issue.type,
|
|
1006
|
+
state: issue.state,
|
|
1007
|
+
created: isoDate(issue.createdAt),
|
|
1008
|
+
url: issue.url,
|
|
1009
|
+
reactions: issue.reactions,
|
|
1010
|
+
comments: issue.comments
|
|
1011
|
+
};
|
|
1012
|
+
if (issue.resolvedIn) fmFields.resolvedIn = issue.resolvedIn;
|
|
1013
|
+
if (issue.labels.length > 0) fmFields.labels = `[${issue.labels.join(", ")}]`;
|
|
1014
|
+
const lines = [
|
|
1015
|
+
buildFrontmatter(fmFields),
|
|
1016
|
+
"",
|
|
1017
|
+
`# ${issue.title}`
|
|
1018
|
+
];
|
|
1019
|
+
if (issue.body) {
|
|
1020
|
+
const body = truncateBody$1(issue.body, limit);
|
|
1021
|
+
lines.push("", body);
|
|
865
1022
|
}
|
|
866
|
-
if (
|
|
867
|
-
|
|
868
|
-
const
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
const
|
|
872
|
-
|
|
1023
|
+
if (issue.topComments.length > 0) {
|
|
1024
|
+
lines.push("", "---", "", "## Top Comments");
|
|
1025
|
+
for (const c of issue.topComments) {
|
|
1026
|
+
const reactions = c.reactions > 0 ? ` (+${c.reactions})` : "";
|
|
1027
|
+
const maintainer = c.isMaintainer ? " [maintainer]" : "";
|
|
1028
|
+
const commentBody = truncateBody$1(c.body, 600);
|
|
1029
|
+
lines.push("", `**@${c.author}**${maintainer}${reactions}:`, "", commentBody);
|
|
873
1030
|
}
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
return releases.map((r) => ({
|
|
877
|
-
path: `releases/blog-${r.version}.md`,
|
|
878
|
-
content: formatBlogRelease(r)
|
|
879
|
-
}));
|
|
1031
|
+
}
|
|
1032
|
+
return lines.join("\n");
|
|
880
1033
|
}
|
|
881
1034
|
/**
|
|
882
|
-
*
|
|
883
|
-
|
|
884
|
-
/**
|
|
885
|
-
* Crawl a URL pattern and return docs as cached doc format.
|
|
886
|
-
* Uses HTTP crawler (no browser needed) with sitemap discovery + glob filtering.
|
|
887
|
-
*
|
|
888
|
-
* @param url - URL with optional glob pattern (e.g. 'https://example.com/docs/**')
|
|
889
|
-
* @param onProgress - Optional progress callback
|
|
890
|
-
* @param maxPages - Max pages to crawl (default 200)
|
|
1035
|
+
* Generate a summary index of all issues for quick LLM scanning.
|
|
1036
|
+
* Groups by type so the LLM can quickly find bugs vs questions.
|
|
891
1037
|
*/
|
|
892
|
-
|
|
893
|
-
const
|
|
894
|
-
|
|
895
|
-
const
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
});
|
|
1038
|
+
function generateIssueIndex(issues) {
|
|
1039
|
+
const byType = /* @__PURE__ */ new Map();
|
|
1040
|
+
for (const issue of issues) mapInsert(byType, issue.type, () => []).push(issue);
|
|
1041
|
+
const typeLabels = {
|
|
1042
|
+
bug: "Bugs & Regressions",
|
|
1043
|
+
question: "Questions & Usage Help",
|
|
1044
|
+
docs: "Documentation",
|
|
1045
|
+
feature: "Feature Requests",
|
|
1046
|
+
other: "Other"
|
|
1047
|
+
};
|
|
1048
|
+
const typeOrder = [
|
|
1049
|
+
"bug",
|
|
1050
|
+
"question",
|
|
1051
|
+
"docs",
|
|
1052
|
+
"other",
|
|
1053
|
+
"feature"
|
|
1054
|
+
];
|
|
1055
|
+
const sections = [
|
|
1056
|
+
[
|
|
1057
|
+
"---",
|
|
1058
|
+
`total: ${issues.length}`,
|
|
1059
|
+
`open: ${issues.filter((i) => i.state === "open").length}`,
|
|
1060
|
+
`closed: ${issues.filter((i) => i.state !== "open").length}`,
|
|
1061
|
+
"---"
|
|
1062
|
+
].join("\n"),
|
|
1063
|
+
"",
|
|
1064
|
+
"# Issues Index",
|
|
1065
|
+
""
|
|
1066
|
+
];
|
|
1067
|
+
for (const type of typeOrder) {
|
|
1068
|
+
const group = byType.get(type);
|
|
1069
|
+
if (!group?.length) continue;
|
|
1070
|
+
sections.push(`## ${typeLabels[type]} (${group.length})`, "");
|
|
1071
|
+
for (const issue of group) {
|
|
1072
|
+
const reactions = issue.reactions > 0 ? ` (+${issue.reactions})` : "";
|
|
1073
|
+
const state = issue.state === "open" ? "" : " [closed]";
|
|
1074
|
+
const resolved = issue.resolvedIn ? ` [fixed in ${issue.resolvedIn}]` : "";
|
|
1075
|
+
const date = isoDate(issue.createdAt);
|
|
1076
|
+
sections.push(`- [#${issue.number}](./issue-${issue.number}.md): ${issue.title}${reactions}${state}${resolved} (${date})`);
|
|
1077
|
+
}
|
|
1078
|
+
sections.push("");
|
|
925
1079
|
}
|
|
926
|
-
|
|
927
|
-
return docs;
|
|
928
|
-
}
|
|
929
|
-
/** Append glob pattern to a docs URL for crawling */
|
|
930
|
-
function toCrawlPattern(docsUrl) {
|
|
931
|
-
return `${docsUrl.replace(/\/+$/, "")}/**`;
|
|
1080
|
+
return sections.join("\n");
|
|
932
1081
|
}
|
|
1082
|
+
//#endregion
|
|
1083
|
+
//#region src/sources/discussions.ts
|
|
933
1084
|
/**
|
|
934
1085
|
* GitHub discussions fetching via gh CLI GraphQL
|
|
935
1086
|
* Prioritizes Q&A and Help categories, includes accepted answers
|
|
@@ -1163,6 +1314,8 @@ function generateDiscussionIndex(discussions) {
|
|
|
1163
1314
|
}
|
|
1164
1315
|
return sections.join("\n");
|
|
1165
1316
|
}
|
|
1317
|
+
//#endregion
|
|
1318
|
+
//#region src/sources/docs.ts
|
|
1166
1319
|
/**
|
|
1167
1320
|
* Docs index generation — creates _INDEX.md for docs directory
|
|
1168
1321
|
*/
|
|
@@ -1215,6 +1368,8 @@ function generateDocsIndex(docs) {
|
|
|
1215
1368
|
}
|
|
1216
1369
|
return sections.join("\n");
|
|
1217
1370
|
}
|
|
1371
|
+
//#endregion
|
|
1372
|
+
//#region src/sources/entries.ts
|
|
1218
1373
|
/**
|
|
1219
1374
|
* Globs .d.ts type definition files from a package for search indexing.
|
|
1220
1375
|
* Only types — source code is too verbose.
|
|
@@ -1275,6 +1430,8 @@ async function resolveEntryFiles(packageDir) {
|
|
|
1275
1430
|
}
|
|
1276
1431
|
return entries;
|
|
1277
1432
|
}
|
|
1433
|
+
//#endregion
|
|
1434
|
+
//#region src/sources/git-skills.ts
|
|
1278
1435
|
/**
|
|
1279
1436
|
* Git repo skill source — parse inputs + fetch pre-authored skills from repos
|
|
1280
1437
|
*
|
|
@@ -1430,7 +1587,8 @@ async function downloadGitHubSkills(owner, repo, ref, skillPath, onProgress) {
|
|
|
1430
1587
|
onProgress?.(`Downloading ${owner}/${repo}/${skillPath}@${ref}`);
|
|
1431
1588
|
const { dir } = await downloadTemplate(`github:${owner}/${repo}/${skillPath}#${ref}`, {
|
|
1432
1589
|
dir: tempDir,
|
|
1433
|
-
force: true
|
|
1590
|
+
force: true,
|
|
1591
|
+
auth: getGitHubToken() || void 0
|
|
1434
1592
|
});
|
|
1435
1593
|
const skill = readLocalSkill(dir, skillPath);
|
|
1436
1594
|
return skill ? [skill] : [];
|
|
@@ -1439,7 +1597,8 @@ async function downloadGitHubSkills(owner, repo, ref, skillPath, onProgress) {
|
|
|
1439
1597
|
try {
|
|
1440
1598
|
const { dir } = await downloadTemplate(`github:${owner}/${repo}/skills#${ref}`, {
|
|
1441
1599
|
dir: tempDir,
|
|
1442
|
-
force: true
|
|
1600
|
+
force: true,
|
|
1601
|
+
auth: getGitHubToken() || void 0
|
|
1443
1602
|
});
|
|
1444
1603
|
const skills = [];
|
|
1445
1604
|
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
@@ -1452,7 +1611,7 @@ async function downloadGitHubSkills(owner, repo, ref, skillPath, onProgress) {
|
|
|
1452
1611
|
return skills;
|
|
1453
1612
|
}
|
|
1454
1613
|
} catch {}
|
|
1455
|
-
const content = await
|
|
1614
|
+
const content = await fetchGitHubRaw(`https://raw.githubusercontent.com/${owner}/${repo}/${ref}/SKILL.md`);
|
|
1456
1615
|
if (content) {
|
|
1457
1616
|
const fm = parseSkillFrontmatterName(content);
|
|
1458
1617
|
onProgress?.("Found 1 skill");
|
|
@@ -1521,6 +1680,8 @@ async function fetchGitLabSkills(source, onProgress) {
|
|
|
1521
1680
|
});
|
|
1522
1681
|
}
|
|
1523
1682
|
}
|
|
1683
|
+
//#endregion
|
|
1684
|
+
//#region src/sources/llms.ts
|
|
1524
1685
|
/**
|
|
1525
1686
|
* Check for llms.txt at a docs URL, returns the llms.txt URL if found
|
|
1526
1687
|
*/
|
|
@@ -1612,15 +1773,27 @@ function extractSections(content, patterns) {
|
|
|
1612
1773
|
if (sections.length === 0) return null;
|
|
1613
1774
|
return sections.join("\n\n---\n\n");
|
|
1614
1775
|
}
|
|
1776
|
+
//#endregion
|
|
1777
|
+
//#region src/sources/github.ts
|
|
1615
1778
|
/** Minimum git-doc file count to prefer over llms.txt */
|
|
1616
1779
|
const MIN_GIT_DOCS = 5;
|
|
1617
1780
|
/** True when git-docs exist but are too few to be useful (< MIN_GIT_DOCS) */
|
|
1618
|
-
const isShallowGitDocs = (n) => n > 0 && n <
|
|
1781
|
+
const isShallowGitDocs = (n) => n > 0 && n < 5;
|
|
1619
1782
|
/**
|
|
1620
|
-
* List files at a git ref
|
|
1783
|
+
* List files at a git ref. Tries ungh.cc first (fast, no rate limits),
|
|
1784
|
+
* falls back to GitHub API for private repos.
|
|
1621
1785
|
*/
|
|
1622
1786
|
async function listFilesAtRef(owner, repo, ref) {
|
|
1623
|
-
|
|
1787
|
+
if (!isKnownPrivateRepo(owner, repo)) {
|
|
1788
|
+
const data = await $fetch(`https://ungh.cc/repos/${owner}/${repo}/files/${ref}`).catch(() => null);
|
|
1789
|
+
if (data?.files?.length) return data.files.map((f) => f.path);
|
|
1790
|
+
}
|
|
1791
|
+
const tree = await ghApi(`repos/${owner}/${repo}/git/trees/${ref}?recursive=1`);
|
|
1792
|
+
if (tree?.tree?.length) {
|
|
1793
|
+
markRepoPrivate(owner, repo);
|
|
1794
|
+
return tree.tree.map((f) => f.path);
|
|
1795
|
+
}
|
|
1796
|
+
return [];
|
|
1624
1797
|
}
|
|
1625
1798
|
/**
|
|
1626
1799
|
* Find git tag for a version by checking if ungh can list files at that ref.
|
|
@@ -1658,13 +1831,29 @@ async function findGitTag(owner, repo, version, packageName, branchHint) {
|
|
|
1658
1831
|
return null;
|
|
1659
1832
|
}
|
|
1660
1833
|
/**
|
|
1661
|
-
*
|
|
1662
|
-
|
|
1834
|
+
* Fetch releases from ungh.cc first, fall back to GitHub API for private repos.
|
|
1835
|
+
*/
|
|
1836
|
+
async function fetchUnghReleases(owner, repo) {
|
|
1837
|
+
if (!isKnownPrivateRepo(owner, repo)) {
|
|
1838
|
+
const data = await $fetch(`https://ungh.cc/repos/${owner}/${repo}/releases`).catch(() => null);
|
|
1839
|
+
if (data?.releases?.length) return data.releases;
|
|
1840
|
+
}
|
|
1841
|
+
const raw = await ghApiPaginated(`repos/${owner}/${repo}/releases`);
|
|
1842
|
+
if (raw.length > 0) {
|
|
1843
|
+
markRepoPrivate(owner, repo);
|
|
1844
|
+
return raw.map((r) => ({
|
|
1845
|
+
tag: r.tag_name,
|
|
1846
|
+
publishedAt: r.published_at
|
|
1847
|
+
}));
|
|
1848
|
+
}
|
|
1849
|
+
return [];
|
|
1850
|
+
}
|
|
1851
|
+
/**
|
|
1852
|
+
* Find the latest release tag matching `{packageName}@*`.
|
|
1663
1853
|
*/
|
|
1664
1854
|
async function findLatestReleaseTag(owner, repo, packageName) {
|
|
1665
|
-
const data = await $fetch(`https://ungh.cc/repos/${owner}/${repo}/releases`).catch(() => null);
|
|
1666
1855
|
const prefix = `${packageName}@`;
|
|
1667
|
-
return
|
|
1856
|
+
return (await fetchUnghReleases(owner, repo)).find((r) => r.tag.startsWith(prefix))?.tag ?? null;
|
|
1668
1857
|
}
|
|
1669
1858
|
/**
|
|
1670
1859
|
* Filter file paths by prefix and md/mdx extension
|
|
@@ -1804,7 +1993,7 @@ function discoverDocFiles(allFiles, packageName) {
|
|
|
1804
1993
|
mapInsert(dirGroups, file.slice(0, lastSlash + 1), () => []).push(file);
|
|
1805
1994
|
}
|
|
1806
1995
|
if (dirGroups.size === 0) return null;
|
|
1807
|
-
const scored =
|
|
1996
|
+
const scored = Array.from(dirGroups.entries(), ([dir, files]) => ({
|
|
1808
1997
|
dir,
|
|
1809
1998
|
files,
|
|
1810
1999
|
score: scoreDocDir(dir, files.length)
|
|
@@ -1914,7 +2103,7 @@ async function verifyNpmRepo(owner, repo, packageName) {
|
|
|
1914
2103
|
`packages/${packageName.replace(/^@/, "").replace("/", "-")}/package.json`
|
|
1915
2104
|
];
|
|
1916
2105
|
for (const path of paths) {
|
|
1917
|
-
const text = await
|
|
2106
|
+
const text = await fetchGitHubRaw(`${base}/${path}`);
|
|
1918
2107
|
if (!text) continue;
|
|
1919
2108
|
try {
|
|
1920
2109
|
if (JSON.parse(text).name === packageName) return true;
|
|
@@ -1971,38 +2160,35 @@ async function searchGitHubRepo(packageName) {
|
|
|
1971
2160
|
async function fetchGitHubRepoMeta(owner, repo, packageName) {
|
|
1972
2161
|
const override = packageName ? getDocOverride(packageName) : void 0;
|
|
1973
2162
|
if (override?.homepage) return { homepage: override.homepage };
|
|
1974
|
-
|
|
1975
|
-
const { stdout: json } = spawnSync("gh", [
|
|
1976
|
-
"api",
|
|
1977
|
-
`repos/${owner}/${repo}`,
|
|
1978
|
-
"-q",
|
|
1979
|
-
"{homepage}"
|
|
1980
|
-
], {
|
|
1981
|
-
encoding: "utf-8",
|
|
1982
|
-
timeout: 1e4
|
|
1983
|
-
});
|
|
1984
|
-
if (!json) throw new Error("no output");
|
|
1985
|
-
const data = JSON.parse(json);
|
|
1986
|
-
return data?.homepage ? { homepage: data.homepage } : null;
|
|
1987
|
-
} catch {}
|
|
1988
|
-
const data = await $fetch(`https://api.github.com/repos/${owner}/${repo}`).catch(() => null);
|
|
2163
|
+
const data = await ghApi(`repos/${owner}/${repo}`) ?? await $fetch(`https://api.github.com/repos/${owner}/${repo}`).catch(() => null);
|
|
1989
2164
|
return data?.homepage ? { homepage: data.homepage } : null;
|
|
1990
2165
|
}
|
|
1991
2166
|
/**
|
|
1992
2167
|
* Resolve README URL for a GitHub repo, returns ungh:// pseudo-URL or raw URL
|
|
1993
2168
|
*/
|
|
1994
2169
|
async function fetchReadme(owner, repo, subdir, ref) {
|
|
1995
|
-
const
|
|
1996
|
-
if ((
|
|
2170
|
+
const branch = ref || "main";
|
|
2171
|
+
if (!isKnownPrivateRepo(owner, repo)) {
|
|
2172
|
+
const unghUrl = subdir ? `https://ungh.cc/repos/${owner}/${repo}/files/${branch}/${subdir}/README.md` : `https://ungh.cc/repos/${owner}/${repo}/readme${ref ? `?ref=${ref}` : ""}`;
|
|
2173
|
+
if ((await $fetch.raw(unghUrl).catch(() => null))?.ok) return `ungh://${owner}/${repo}${subdir ? `/${subdir}` : ""}${ref ? `@${ref}` : ""}`;
|
|
2174
|
+
}
|
|
1997
2175
|
const basePath = subdir ? `${subdir}/` : "";
|
|
1998
2176
|
const branches = ref ? [ref] : ["main", "master"];
|
|
2177
|
+
const token = isKnownPrivateRepo(owner, repo) ? getGitHubToken() : null;
|
|
2178
|
+
const authHeaders = token ? { Authorization: `token ${token}` } : {};
|
|
1999
2179
|
for (const b of branches) for (const filename of [
|
|
2000
2180
|
"README.md",
|
|
2001
2181
|
"Readme.md",
|
|
2002
2182
|
"readme.md"
|
|
2003
2183
|
]) {
|
|
2004
2184
|
const readmeUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${b}/${basePath}${filename}`;
|
|
2005
|
-
if ((await $fetch.raw(readmeUrl).catch(() => null))?.ok) return readmeUrl;
|
|
2185
|
+
if ((await $fetch.raw(readmeUrl, { headers: authHeaders }).catch(() => null))?.ok) return readmeUrl;
|
|
2186
|
+
}
|
|
2187
|
+
const refParam = ref ? `?ref=${ref}` : "";
|
|
2188
|
+
const apiData = await ghApi(subdir ? `repos/${owner}/${repo}/contents/${subdir}/README.md${refParam}` : `repos/${owner}/${repo}/readme${refParam}`);
|
|
2189
|
+
if (apiData?.download_url) {
|
|
2190
|
+
markRepoPrivate(owner, repo);
|
|
2191
|
+
return apiData.download_url;
|
|
2006
2192
|
}
|
|
2007
2193
|
return null;
|
|
2008
2194
|
}
|
|
@@ -2036,6 +2222,7 @@ async function fetchReadmeContent(url) {
|
|
|
2036
2222
|
return text;
|
|
2037
2223
|
}
|
|
2038
2224
|
}
|
|
2225
|
+
if (url.includes("raw.githubusercontent.com")) return fetchGitHubRaw(url);
|
|
2039
2226
|
return fetchText(url);
|
|
2040
2227
|
}
|
|
2041
2228
|
/**
|
|
@@ -2045,34 +2232,14 @@ async function fetchReadmeContent(url) {
|
|
|
2045
2232
|
async function resolveGitHubRepo(owner, repo, onProgress) {
|
|
2046
2233
|
onProgress?.("Fetching repo metadata");
|
|
2047
2234
|
const repoUrl = `https://github.com/${owner}/${repo}`;
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
const { stdout: json } = spawnSync("gh", [
|
|
2052
|
-
"api",
|
|
2053
|
-
`repos/${owner}/${repo}`,
|
|
2054
|
-
"--jq",
|
|
2055
|
-
"{homepage: .homepage, description: .description}"
|
|
2056
|
-
], {
|
|
2057
|
-
encoding: "utf-8",
|
|
2058
|
-
timeout: 1e4
|
|
2059
|
-
});
|
|
2060
|
-
if (json) {
|
|
2061
|
-
const data = JSON.parse(json);
|
|
2062
|
-
homepage = data.homepage || void 0;
|
|
2063
|
-
description = data.description || void 0;
|
|
2064
|
-
}
|
|
2065
|
-
} catch {}
|
|
2066
|
-
if (!homepage && !description) {
|
|
2067
|
-
const data = await $fetch(`https://api.github.com/repos/${owner}/${repo}`).catch(() => null);
|
|
2068
|
-
homepage = data?.homepage || void 0;
|
|
2069
|
-
description = data?.description || void 0;
|
|
2070
|
-
}
|
|
2235
|
+
const meta = await ghApi(`repos/${owner}/${repo}`) ?? await $fetch(`https://api.github.com/repos/${owner}/${repo}`).catch(() => null);
|
|
2236
|
+
const homepage = meta?.homepage || void 0;
|
|
2237
|
+
const description = meta?.description || void 0;
|
|
2071
2238
|
onProgress?.("Fetching latest release");
|
|
2072
|
-
const
|
|
2239
|
+
const releases = await fetchUnghReleases(owner, repo);
|
|
2073
2240
|
let version = "main";
|
|
2074
2241
|
let releasedAt;
|
|
2075
|
-
const latestRelease =
|
|
2242
|
+
const latestRelease = releases[0];
|
|
2076
2243
|
if (latestRelease) {
|
|
2077
2244
|
version = latestRelease.tag.replace(/^v/, "");
|
|
2078
2245
|
releasedAt = latestRelease.publishedAt;
|
|
@@ -2103,6 +2270,8 @@ async function resolveGitHubRepo(owner, repo, onProgress) {
|
|
|
2103
2270
|
llmsUrl
|
|
2104
2271
|
};
|
|
2105
2272
|
}
|
|
2273
|
+
//#endregion
|
|
2274
|
+
//#region src/sources/npm.ts
|
|
2106
2275
|
/**
|
|
2107
2276
|
* Search npm registry for packages matching a query.
|
|
2108
2277
|
* Used as a fallback when direct package lookup fails.
|
|
@@ -2545,6 +2714,7 @@ function getInstalledSkillVersion(skillDir) {
|
|
|
2545
2714
|
if (!existsSync(skillPath)) return null;
|
|
2546
2715
|
return readFileSync(skillPath, "utf-8").match(/^version:\s*"?([^"\n]+)"?/m)?.[1] || null;
|
|
2547
2716
|
}
|
|
2548
|
-
|
|
2717
|
+
//#endregion
|
|
2718
|
+
export { isGitHubRepoUrl as $, parseGitSkillInput as A, isGhAvailable as B, downloadLlmsDocs as C, normalizeLlmsLinks as D, fetchLlmsUrl as E, formatDiscussionAsMarkdown as F, fetchReleaseNotes as G, toCrawlPattern as H, generateDiscussionIndex as I, parseSemver as J, generateReleaseIndex as K, fetchGitHubIssues as L, resolveEntryFiles as M, generateDocsIndex as N, parseMarkdownLinks as O, fetchGitHubDiscussions as P, fetchText as Q, formatIssueAsMarkdown as R, validateGitDocsWithLlms as S, fetchLlmsTxt as T, fetchBlogReleases as U, fetchCrawledDocs as V, compareSemver as W, extractBranchHint as X, $fetch as Y, fetchGitHubRaw as Z, fetchReadme as _, getInstalledSkillVersion as a, isShallowGitDocs as b, readLocalPackageInfo as c, resolvePackageDocs as d, normalizeRepoUrl as et, resolvePackageDocsWithAttempts as f, fetchGitHubRepoMeta as g, fetchGitDocs as h, fetchPkgDist as i, parseSkillFrontmatterName as j, fetchGitSkills as k, resolveInstalledVersion as l, MIN_GIT_DOCS as m, fetchNpmPackage as n, parsePackageSpec as nt, parseVersionSpecifier as o, searchNpmPackages as p, isPrerelease as q, fetchNpmRegistryMeta as r, verifyUrl as rt, readLocalDependencies as s, fetchLatestVersion as t, parseGitHubUrl as tt, resolveLocalPackageDocs as u, fetchReadmeContent as v, extractSections as w, resolveGitHubRepo as x, filterFrameworkDocs as y, generateIssueIndex as z };
|
|
2549
2719
|
|
|
2550
2720
|
//# sourceMappingURL=sources.mjs.map
|