skilld 1.6.2 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -20
- package/dist/_chunks/agent.mjs +2 -1
- package/dist/_chunks/agent.mjs.map +1 -1
- package/dist/_chunks/assemble.mjs +1 -1
- package/dist/_chunks/author-group.mjs +17 -0
- package/dist/_chunks/author-group.mjs.map +1 -0
- package/dist/_chunks/author.mjs +8 -6
- package/dist/_chunks/author.mjs.map +1 -1
- package/dist/_chunks/cache.mjs +1 -1
- package/dist/_chunks/cache2.mjs +1 -1
- package/dist/_chunks/cli-helpers.mjs +3 -119
- package/dist/_chunks/cli-helpers.mjs.map +1 -1
- package/dist/_chunks/config.mjs +119 -27
- package/dist/_chunks/config.mjs.map +1 -1
- package/dist/_chunks/core.mjs +1 -1
- package/dist/_chunks/embedding-cache2.mjs +1 -1
- package/dist/_chunks/index.d.mts.map +1 -1
- package/dist/_chunks/index3.d.mts +79 -78
- package/dist/_chunks/index3.d.mts.map +1 -1
- package/dist/_chunks/install.mjs +85 -535
- package/dist/_chunks/install.mjs.map +1 -1
- package/dist/_chunks/install2.mjs +554 -0
- package/dist/_chunks/install2.mjs.map +1 -0
- package/dist/_chunks/lockfile.mjs +1 -0
- package/dist/_chunks/lockfile.mjs.map +1 -1
- package/dist/_chunks/package-registry.mjs +465 -0
- package/dist/_chunks/package-registry.mjs.map +1 -0
- package/dist/_chunks/prefix.mjs +108 -0
- package/dist/_chunks/prefix.mjs.map +1 -0
- package/dist/_chunks/prepare.mjs +6 -2
- package/dist/_chunks/prepare.mjs.map +1 -1
- package/dist/_chunks/prepare2.mjs +1 -1
- package/dist/_chunks/prompts.mjs +5 -99
- package/dist/_chunks/prompts.mjs.map +1 -1
- package/dist/_chunks/search-helpers.mjs +99 -0
- package/dist/_chunks/search-helpers.mjs.map +1 -0
- package/dist/_chunks/search-interactive.mjs +1 -1
- package/dist/_chunks/search-interactive.mjs.map +1 -1
- package/dist/_chunks/search.mjs +219 -1
- package/dist/_chunks/search.mjs.map +1 -0
- package/dist/_chunks/shared.mjs +1 -463
- package/dist/_chunks/shared.mjs.map +1 -1
- package/dist/_chunks/skills.mjs +1 -1
- package/dist/_chunks/sources.mjs +1177 -988
- package/dist/_chunks/sources.mjs.map +1 -1
- package/dist/_chunks/sync-registry.mjs +59 -0
- package/dist/_chunks/sync-registry.mjs.map +1 -0
- package/dist/_chunks/sync-shared2.mjs +10 -7
- package/dist/_chunks/sync-shared2.mjs.map +1 -1
- package/dist/_chunks/sync.mjs +208 -99
- package/dist/_chunks/sync.mjs.map +1 -1
- package/dist/_chunks/sync2.mjs +1 -1
- package/dist/_chunks/uninstall.mjs +3 -2
- package/dist/_chunks/uninstall.mjs.map +1 -1
- package/dist/_chunks/upload.mjs +152 -0
- package/dist/_chunks/upload.mjs.map +1 -0
- package/dist/_chunks/validate.mjs +1 -1
- package/dist/_chunks/version.mjs +30 -0
- package/dist/_chunks/version.mjs.map +1 -0
- package/dist/_chunks/wizard.mjs +2 -1
- package/dist/_chunks/wizard.mjs.map +1 -1
- package/dist/agent/index.mjs +2 -1
- package/dist/cache/index.mjs +1 -1
- package/dist/cli.mjs +48 -20
- package/dist/cli.mjs.map +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/index.mjs +2 -2
- package/dist/sources/index.d.mts +2 -2
- package/dist/sources/index.mjs +3 -3
- package/dist/types.d.mts +1 -1
- package/package.json +11 -12
- package/dist/_chunks/search2.mjs +0 -310
- package/dist/_chunks/search2.mjs.map +0 -1
package/dist/_chunks/sources.mjs
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { t as getCacheDir } from "./version.mjs";
|
|
2
2
|
import { i as readPackageJsonSafe } from "./package-json.mjs";
|
|
3
3
|
import { t as yamlEscape } from "./yaml.mjs";
|
|
4
4
|
import { i as parseFrontmatter, n as extractLinks, r as extractTitle, t as extractDescription } from "./markdown.mjs";
|
|
5
|
-
import {
|
|
5
|
+
import { n as getCrawlUrl, r as getDocOverride, t as getBlogPreset } from "./package-registry.mjs";
|
|
6
|
+
import { r as mapInsert } from "./shared.mjs";
|
|
6
7
|
import { tmpdir } from "node:os";
|
|
7
8
|
import { basename, dirname, join, resolve } from "pathe";
|
|
8
9
|
import { createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, rmSync } from "node:fs";
|
|
@@ -10,10 +11,10 @@ import { htmlToMarkdown } from "mdream";
|
|
|
10
11
|
import pLimit from "p-limit";
|
|
11
12
|
import { spawnSync } from "node:child_process";
|
|
12
13
|
import { ofetch } from "ofetch";
|
|
14
|
+
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
13
15
|
import { crawlAndGenerate } from "@mdream/crawl";
|
|
14
16
|
import { glob } from "tinyglobby";
|
|
15
17
|
import { downloadTemplate } from "giget";
|
|
16
|
-
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
17
18
|
import { Writable } from "node:stream";
|
|
18
19
|
import { resolvePathSync } from "mlly";
|
|
19
20
|
const BOT_USERS = new Set([
|
|
@@ -110,12 +111,36 @@ async function ghApiPaginated(endpoint) {
|
|
|
110
111
|
}
|
|
111
112
|
return results;
|
|
112
113
|
}
|
|
114
|
+
const SKILLD_USER_AGENT = "skilld/1.0 (+https://github.com/harlan-zw/skilld)";
|
|
113
115
|
const $fetch = ofetch.create({
|
|
114
116
|
retry: 3,
|
|
115
|
-
retryDelay:
|
|
117
|
+
retryDelay: 1e3,
|
|
118
|
+
retryStatusCodes: [
|
|
119
|
+
408,
|
|
120
|
+
429,
|
|
121
|
+
500,
|
|
122
|
+
502,
|
|
123
|
+
503,
|
|
124
|
+
504
|
|
125
|
+
],
|
|
116
126
|
timeout: 15e3,
|
|
117
|
-
headers: { "User-Agent":
|
|
127
|
+
headers: { "User-Agent": SKILLD_USER_AGENT }
|
|
118
128
|
});
|
|
129
|
+
function createRateLimitedRunner(intervalMs) {
|
|
130
|
+
let queue = Promise.resolve();
|
|
131
|
+
let lastRunAt = 0;
|
|
132
|
+
return async function runRateLimited(task) {
|
|
133
|
+
const run = async () => {
|
|
134
|
+
const waitMs = intervalMs - (Date.now() - lastRunAt);
|
|
135
|
+
if (waitMs > 0) await new Promise((resolve) => setTimeout(resolve, waitMs));
|
|
136
|
+
lastRunAt = Date.now();
|
|
137
|
+
return task();
|
|
138
|
+
};
|
|
139
|
+
const request = queue.then(run, run);
|
|
140
|
+
queue = request.then(() => void 0, () => void 0);
|
|
141
|
+
return request;
|
|
142
|
+
};
|
|
143
|
+
}
|
|
119
144
|
async function fetchText(url) {
|
|
120
145
|
return $fetch(url, { responseType: "text" }).catch(() => null);
|
|
121
146
|
}
|
|
@@ -175,6 +200,20 @@ function isGitHubRepoUrl(url) {
|
|
|
175
200
|
return false;
|
|
176
201
|
}
|
|
177
202
|
}
|
|
203
|
+
function isLikelyCodeHostUrl(url) {
|
|
204
|
+
if (!url) return false;
|
|
205
|
+
try {
|
|
206
|
+
const parsed = new URL(url);
|
|
207
|
+
return [
|
|
208
|
+
"github.com",
|
|
209
|
+
"www.github.com",
|
|
210
|
+
"gitlab.com",
|
|
211
|
+
"www.gitlab.com"
|
|
212
|
+
].includes(parsed.hostname);
|
|
213
|
+
} catch {
|
|
214
|
+
return false;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
178
217
|
function parseGitHubUrl(url) {
|
|
179
218
|
const match = url.match(/github\.com\/([^/]+)\/([^/]+?)(?:\.git)?(?:[/#]|$)/);
|
|
180
219
|
if (!match) return null;
|
|
@@ -456,99 +495,6 @@ async function fetchBlogReleases(packageName, installedVersion) {
|
|
|
456
495
|
content: formatBlogRelease(r)
|
|
457
496
|
}));
|
|
458
497
|
}
|
|
459
|
-
async function fetchCrawledDocs(url, onProgress, maxPages = 200) {
|
|
460
|
-
const outputDir = join(tmpdir(), "skilld-crawl", Date.now().toString());
|
|
461
|
-
onProgress?.(`Crawling ${url}`);
|
|
462
|
-
const userLang = getUserLang();
|
|
463
|
-
const foreignUrls = /* @__PURE__ */ new Set();
|
|
464
|
-
const doCrawl = () => crawlAndGenerate({
|
|
465
|
-
urls: [url],
|
|
466
|
-
outputDir,
|
|
467
|
-
driver: "http",
|
|
468
|
-
generateLlmsTxt: false,
|
|
469
|
-
generateIndividualMd: true,
|
|
470
|
-
maxRequestsPerCrawl: maxPages,
|
|
471
|
-
onPage: (page) => {
|
|
472
|
-
const lang = extractHtmlLang(page.html);
|
|
473
|
-
if (lang && !lang.startsWith("en") && !lang.startsWith(userLang)) foreignUrls.add(page.url);
|
|
474
|
-
}
|
|
475
|
-
}, (progress) => {
|
|
476
|
-
if (progress.crawling.status === "processing" && progress.crawling.total > 0) onProgress?.(`Crawling ${progress.crawling.processed}/${progress.crawling.total} pages`);
|
|
477
|
-
});
|
|
478
|
-
let results = await doCrawl().catch((err) => {
|
|
479
|
-
onProgress?.(`Crawl failed: ${err?.message || err}`);
|
|
480
|
-
return [];
|
|
481
|
-
});
|
|
482
|
-
if (results.length === 0) {
|
|
483
|
-
onProgress?.("Retrying crawl");
|
|
484
|
-
results = await doCrawl().catch(() => []);
|
|
485
|
-
}
|
|
486
|
-
rmSync(outputDir, {
|
|
487
|
-
recursive: true,
|
|
488
|
-
force: true
|
|
489
|
-
});
|
|
490
|
-
const docs = [];
|
|
491
|
-
let localeFiltered = 0;
|
|
492
|
-
for (const result of results) {
|
|
493
|
-
if (!result.success || !result.content) continue;
|
|
494
|
-
if (foreignUrls.has(result.url)) {
|
|
495
|
-
localeFiltered++;
|
|
496
|
-
continue;
|
|
497
|
-
}
|
|
498
|
-
const segments = (new URL(result.url).pathname.replace(/\/$/, "") || "/index").split("/").filter(Boolean);
|
|
499
|
-
if (isForeignPathPrefix(segments[0], userLang)) {
|
|
500
|
-
localeFiltered++;
|
|
501
|
-
continue;
|
|
502
|
-
}
|
|
503
|
-
const path = `docs/${segments.join("/")}.md`;
|
|
504
|
-
docs.push({
|
|
505
|
-
path,
|
|
506
|
-
content: result.content
|
|
507
|
-
});
|
|
508
|
-
}
|
|
509
|
-
if (localeFiltered > 0) onProgress?.(`Filtered ${localeFiltered} foreign locale pages`);
|
|
510
|
-
onProgress?.(`Crawled ${docs.length} pages`);
|
|
511
|
-
return docs;
|
|
512
|
-
}
|
|
513
|
-
const HTML_LANG_RE = /<html[^>]*\slang=["']([^"']+)["']/i;
|
|
514
|
-
function extractHtmlLang(html) {
|
|
515
|
-
return HTML_LANG_RE.exec(html)?.[1]?.toLowerCase();
|
|
516
|
-
}
|
|
517
|
-
const LOCALE_CODES = new Set([
|
|
518
|
-
"ar",
|
|
519
|
-
"de",
|
|
520
|
-
"es",
|
|
521
|
-
"fr",
|
|
522
|
-
"id",
|
|
523
|
-
"it",
|
|
524
|
-
"ja",
|
|
525
|
-
"ko",
|
|
526
|
-
"nl",
|
|
527
|
-
"pl",
|
|
528
|
-
"pt",
|
|
529
|
-
"pt-br",
|
|
530
|
-
"ru",
|
|
531
|
-
"th",
|
|
532
|
-
"tr",
|
|
533
|
-
"uk",
|
|
534
|
-
"vi",
|
|
535
|
-
"zh",
|
|
536
|
-
"zh-cn",
|
|
537
|
-
"zh-tw"
|
|
538
|
-
]);
|
|
539
|
-
function isForeignPathPrefix(segment, userLang) {
|
|
540
|
-
if (!segment) return false;
|
|
541
|
-
const lower = segment.toLowerCase();
|
|
542
|
-
if (lower === "en" || lower.startsWith(userLang)) return false;
|
|
543
|
-
return LOCALE_CODES.has(lower);
|
|
544
|
-
}
|
|
545
|
-
function getUserLang() {
|
|
546
|
-
const code = (process.env.LC_ALL || process.env.LANG || process.env.LANGUAGE || "").split(/[_.:-]/)[0]?.toLowerCase() || "";
|
|
547
|
-
return code.length >= 2 ? code.slice(0, 2) : "en";
|
|
548
|
-
}
|
|
549
|
-
function toCrawlPattern(docsUrl) {
|
|
550
|
-
return `${docsUrl.replace(/\/+$/, "")}/**`;
|
|
551
|
-
}
|
|
552
498
|
let _ghAvailable;
|
|
553
499
|
function isGhAvailable() {
|
|
554
500
|
if (_ghAvailable !== void 0) return _ghAvailable;
|
|
@@ -870,992 +816,1235 @@ function generateIssueIndex(issues) {
|
|
|
870
816
|
}
|
|
871
817
|
return sections.join("\n");
|
|
872
818
|
}
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
"support"
|
|
878
|
-
]);
|
|
879
|
-
const LOW_VALUE_CATEGORIES = new Set([
|
|
880
|
-
"show and tell",
|
|
881
|
-
"ideas",
|
|
882
|
-
"polls"
|
|
883
|
-
]);
|
|
884
|
-
const TITLE_NOISE_RE = /looking .*(?:developer|engineer|freelanc)|hiring|job post|guide me to (?:complete|finish|build)|help me (?:complete|finish|build)|seeking .* tutorial|recommend.* course/i;
|
|
885
|
-
const MIN_DISCUSSION_SCORE = 3;
|
|
886
|
-
function scoreComment(c) {
|
|
887
|
-
return (c.isMaintainer ? 3 : 1) * (hasCodeBlock(c.body) ? 2 : 1) * (1 + c.reactions);
|
|
819
|
+
async function fetchLlmsUrl(docsUrl) {
|
|
820
|
+
const llmsUrl = `${new URL(docsUrl).origin}/llms.txt`;
|
|
821
|
+
if (await verifyUrl(llmsUrl)) return llmsUrl;
|
|
822
|
+
return null;
|
|
888
823
|
}
|
|
889
|
-
function
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
...d.topComments.map((c) => c.body)
|
|
897
|
-
].join("\n"))) score += 3;
|
|
898
|
-
score += Math.min(d.upvoteCount, 5);
|
|
899
|
-
if (d.answer) {
|
|
900
|
-
score += 2;
|
|
901
|
-
if (d.answer.length > 100) score += 1;
|
|
902
|
-
}
|
|
903
|
-
if (d.topComments.some((c) => c.isMaintainer)) score += 2;
|
|
904
|
-
if (d.topComments.some((c) => c.reactions > 0)) score += 1;
|
|
905
|
-
return score;
|
|
824
|
+
async function fetchLlmsTxt(url) {
|
|
825
|
+
const content = await fetchText(url);
|
|
826
|
+
if (!content || content.length < 50) return null;
|
|
827
|
+
return {
|
|
828
|
+
raw: content,
|
|
829
|
+
links: parseMarkdownLinks(content)
|
|
830
|
+
};
|
|
906
831
|
}
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
cutoff.setMonth(cutoff.getMonth() + 6);
|
|
912
|
-
if (cutoff < /* @__PURE__ */ new Date()) return [];
|
|
913
|
-
}
|
|
832
|
+
function parseMarkdownLinks(content) {
|
|
833
|
+
return extractLinks(content).filter((l) => l.url.endsWith(".md"));
|
|
834
|
+
}
|
|
835
|
+
function isSafeUrl(url) {
|
|
914
836
|
try {
|
|
915
|
-
const
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
"-f",
|
|
923
|
-
`repo=${repo}`
|
|
924
|
-
], {
|
|
925
|
-
encoding: "utf-8",
|
|
926
|
-
maxBuffer: 10 * 1024 * 1024
|
|
927
|
-
});
|
|
928
|
-
if (!result) return [];
|
|
929
|
-
const nodes = JSON.parse(result)?.data?.repository?.discussions?.nodes;
|
|
930
|
-
if (!Array.isArray(nodes)) return [];
|
|
931
|
-
const fromTs = fromDate ? new Date(fromDate).getTime() : null;
|
|
932
|
-
return nodes.filter((d) => d.author && !BOT_USERS.has(d.author.login)).filter((d) => {
|
|
933
|
-
const cat = (d.category?.name || "").toLowerCase();
|
|
934
|
-
return !LOW_VALUE_CATEGORIES.has(cat);
|
|
935
|
-
}).filter((d) => !fromTs || new Date(d.createdAt).getTime() >= fromTs).map((d) => {
|
|
936
|
-
let answer;
|
|
937
|
-
if (d.answer?.body) {
|
|
938
|
-
const isMaintainer = [
|
|
939
|
-
"OWNER",
|
|
940
|
-
"MEMBER",
|
|
941
|
-
"COLLABORATOR"
|
|
942
|
-
].includes(d.answer.authorAssociation);
|
|
943
|
-
const author = d.answer.author?.login;
|
|
944
|
-
answer = `${isMaintainer && author ? `**@${author}** [maintainer]:\n\n` : ""}${d.answer.body}`;
|
|
945
|
-
}
|
|
946
|
-
const comments = (d.comments?.nodes || []).filter((c) => c.author && !BOT_USERS.has(c.author.login)).filter((c) => !COMMENT_NOISE_RE.test((c.body || "").trim())).map((c) => {
|
|
947
|
-
const isMaintainer = [
|
|
948
|
-
"OWNER",
|
|
949
|
-
"MEMBER",
|
|
950
|
-
"COLLABORATOR"
|
|
951
|
-
].includes(c.authorAssociation);
|
|
952
|
-
return {
|
|
953
|
-
body: c.body || "",
|
|
954
|
-
author: c.author.login,
|
|
955
|
-
reactions: c.reactions?.totalCount || 0,
|
|
956
|
-
isMaintainer
|
|
957
|
-
};
|
|
958
|
-
}).sort((a, b) => scoreComment(b) - scoreComment(a)).slice(0, 3);
|
|
959
|
-
return {
|
|
960
|
-
number: d.number,
|
|
961
|
-
title: d.title,
|
|
962
|
-
body: d.body || "",
|
|
963
|
-
category: d.category?.name || "",
|
|
964
|
-
createdAt: d.createdAt,
|
|
965
|
-
url: d.url,
|
|
966
|
-
upvoteCount: d.upvoteCount || 0,
|
|
967
|
-
comments: d.comments?.totalCount || 0,
|
|
968
|
-
isMaintainer: [
|
|
969
|
-
"OWNER",
|
|
970
|
-
"MEMBER",
|
|
971
|
-
"COLLABORATOR"
|
|
972
|
-
].includes(d.authorAssociation),
|
|
973
|
-
answer,
|
|
974
|
-
topComments: comments
|
|
975
|
-
};
|
|
976
|
-
}).map((d) => ({
|
|
977
|
-
d,
|
|
978
|
-
score: scoreDiscussion(d)
|
|
979
|
-
})).filter(({ score }) => score >= MIN_DISCUSSION_SCORE).sort((a, b) => {
|
|
980
|
-
const aHigh = HIGH_VALUE_CATEGORIES.has(a.d.category.toLowerCase()) ? 1 : 0;
|
|
981
|
-
const bHigh = HIGH_VALUE_CATEGORIES.has(b.d.category.toLowerCase()) ? 1 : 0;
|
|
982
|
-
if (aHigh !== bHigh) return bHigh - aHigh;
|
|
983
|
-
return b.score - a.score;
|
|
984
|
-
}).slice(0, limit).map(({ d }) => d);
|
|
837
|
+
const parsed = new URL(url);
|
|
838
|
+
if (parsed.protocol !== "https:") return false;
|
|
839
|
+
const host = parsed.hostname;
|
|
840
|
+
if (host === "localhost" || host === "0.0.0.0" || host === "[::1]") return false;
|
|
841
|
+
if (/^(?:127\.|10\.|172\.(?:1[6-9]|2\d|3[01])\.|192\.168\.|169\.254\.)/.test(host)) return false;
|
|
842
|
+
if (/^\[(?:f[cd]|fe[89ab]|::ffff:)/i.test(host)) return false;
|
|
843
|
+
return true;
|
|
985
844
|
} catch {
|
|
986
|
-
return
|
|
845
|
+
return false;
|
|
987
846
|
}
|
|
988
847
|
}
|
|
989
|
-
function
|
|
990
|
-
const
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
`# ${d.title}`
|
|
1005
|
-
];
|
|
1006
|
-
if (d.body) lines.push("", truncateBody(d.body, bodyLimit));
|
|
1007
|
-
if (d.answer) lines.push("", "---", "", "## Accepted Answer", "", truncateBody(d.answer, 1e3));
|
|
1008
|
-
else if (d.topComments.length > 0) {
|
|
1009
|
-
lines.push("", "---", "", "## Top Comments");
|
|
1010
|
-
for (const c of d.topComments) {
|
|
1011
|
-
const reactions = c.reactions > 0 ? ` (+${c.reactions})` : "";
|
|
1012
|
-
const maintainer = c.isMaintainer ? " [maintainer]" : "";
|
|
1013
|
-
lines.push("", `**@${c.author}**${maintainer}${reactions}:`, "", truncateBody(c.body, 600));
|
|
1014
|
-
}
|
|
1015
|
-
}
|
|
1016
|
-
return lines.join("\n");
|
|
848
|
+
async function downloadLlmsDocs(llmsContent, baseUrl, onProgress) {
|
|
849
|
+
const limit = pLimit(5);
|
|
850
|
+
let completed = 0;
|
|
851
|
+
return (await Promise.all(llmsContent.links.map((link) => limit(async () => {
|
|
852
|
+
const url = link.url.startsWith("http") ? link.url : `${baseUrl.replace(/\/$/, "")}${link.url.startsWith("/") ? "" : "/"}${link.url}`;
|
|
853
|
+
if (!isSafeUrl(url)) return null;
|
|
854
|
+
const content = await fetchText(url);
|
|
855
|
+
onProgress?.(link.url, ++completed, llmsContent.links.length);
|
|
856
|
+
if (content && content.length > 100) return {
|
|
857
|
+
url: link.url.startsWith("http") ? new URL(link.url).pathname : link.url,
|
|
858
|
+
title: link.title,
|
|
859
|
+
content
|
|
860
|
+
};
|
|
861
|
+
return null;
|
|
862
|
+
})))).filter((d) => d !== null);
|
|
1017
863
|
}
|
|
1018
|
-
function
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
[
|
|
1024
|
-
"---",
|
|
1025
|
-
`total: ${discussions.length}`,
|
|
1026
|
-
`answered: ${answered}`,
|
|
1027
|
-
"---"
|
|
1028
|
-
].join("\n"),
|
|
1029
|
-
"",
|
|
1030
|
-
"# Discussions Index",
|
|
1031
|
-
""
|
|
1032
|
-
];
|
|
1033
|
-
const cats = [...byCategory.keys()].sort((a, b) => {
|
|
1034
|
-
return (HIGH_VALUE_CATEGORIES.has(a.toLowerCase()) ? 0 : 1) - (HIGH_VALUE_CATEGORIES.has(b.toLowerCase()) ? 0 : 1) || a.localeCompare(b);
|
|
1035
|
-
});
|
|
1036
|
-
for (const cat of cats) {
|
|
1037
|
-
const group = byCategory.get(cat);
|
|
1038
|
-
sections.push(`## ${cat} (${group.length})`, "");
|
|
1039
|
-
for (const d of group) {
|
|
1040
|
-
const upvotes = d.upvoteCount > 0 ? ` (+${d.upvoteCount})` : "";
|
|
1041
|
-
const answered = d.answer ? " [answered]" : "";
|
|
1042
|
-
const date = isoDate(d.createdAt);
|
|
1043
|
-
sections.push(`- [#${d.number}](./discussion-${d.number}.md): ${d.title}${upvotes}${answered} (${date})`);
|
|
1044
|
-
}
|
|
1045
|
-
sections.push("");
|
|
864
|
+
function normalizeLlmsLinks(content, baseUrl) {
|
|
865
|
+
let normalized = content;
|
|
866
|
+
if (baseUrl) {
|
|
867
|
+
const escaped = baseUrl.replace(/\/$/, "").replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
868
|
+
normalized = normalized.replace(new RegExp(`\\]\\(${escaped}(/[^)]+\\.md)\\)`, "g"), "](./docs$1)");
|
|
1046
869
|
}
|
|
1047
|
-
|
|
870
|
+
normalized = normalized.replace(/\]\(\/([^)]+\.md)\)/g, "](./docs/$1)");
|
|
871
|
+
return normalized;
|
|
1048
872
|
}
|
|
1049
|
-
function
|
|
1050
|
-
const
|
|
1051
|
-
|
|
1052
|
-
const
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
const
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
const list = byDir.get(dir);
|
|
1060
|
-
if (list) list.push(doc);
|
|
1061
|
-
else byDir.set(dir, [doc]);
|
|
873
|
+
function extractSections(content, patterns) {
|
|
874
|
+
const sections = [];
|
|
875
|
+
const parts = content.split(/\n---\n/);
|
|
876
|
+
for (const part of parts) {
|
|
877
|
+
const urlMatch = part.match(/^url: *(\S.*)$/m);
|
|
878
|
+
if (!urlMatch) continue;
|
|
879
|
+
const url = urlMatch[1];
|
|
880
|
+
if (patterns.some((p) => url.includes(p))) {
|
|
881
|
+
const contentStart = part.indexOf("\n", part.indexOf("url:"));
|
|
882
|
+
if (contentStart > -1) sections.push(part.slice(contentStart + 1));
|
|
1062
883
|
}
|
|
1063
884
|
}
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
const rel = file.path.slice(5);
|
|
1074
|
-
const title = extractTitle(file.content) || rel.replace(/\.md$/, "");
|
|
1075
|
-
const desc = extractDescription(file.content);
|
|
1076
|
-
const descPart = desc ? `: ${desc}` : "";
|
|
1077
|
-
sections.push(`- [${title}](./${rel})${descPart}`);
|
|
885
|
+
if (sections.length === 0) return null;
|
|
886
|
+
return sections.join("\n\n---\n\n");
|
|
887
|
+
}
|
|
888
|
+
const MIN_GIT_DOCS = 5;
|
|
889
|
+
const isShallowGitDocs = (n) => n > 0 && n < 5;
|
|
890
|
+
async function listFilesAtRef(owner, repo, ref) {
|
|
891
|
+
if (!isKnownPrivateRepo(owner, repo)) {
|
|
892
|
+
const data = await $fetch(`https://ungh.cc/repos/${owner}/${repo}/files/${ref}`).catch(() => null);
|
|
893
|
+
if (data?.files?.length) return data.files.map((f) => f.path);
|
|
1078
894
|
}
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
const rel = file.path.slice(5);
|
|
1084
|
-
const title = extractTitle(file.content) || rel.replace(/\.md$/, "").split("/").pop();
|
|
1085
|
-
const desc = extractDescription(file.content);
|
|
1086
|
-
const descPart = desc ? `: ${desc}` : "";
|
|
1087
|
-
sections.push(`- [${title}](./${rel})${descPart}`);
|
|
1088
|
-
}
|
|
1089
|
-
sections.push("");
|
|
895
|
+
const tree = await ghApi(`repos/${owner}/${repo}/git/trees/${ref}?recursive=1`);
|
|
896
|
+
if (tree?.tree?.length) {
|
|
897
|
+
markRepoPrivate(owner, repo);
|
|
898
|
+
return tree.tree.map((f) => f.path);
|
|
1090
899
|
}
|
|
1091
|
-
return
|
|
900
|
+
return [];
|
|
1092
901
|
}
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
"*.global.*",
|
|
1112
|
-
"*.browser.*",
|
|
1113
|
-
"*.map",
|
|
1114
|
-
"*.map.js",
|
|
1115
|
-
"CHANGELOG*",
|
|
1116
|
-
"LICENSE*",
|
|
1117
|
-
"README*"
|
|
1118
|
-
];
|
|
1119
|
-
const MAX_FILE_SIZE = 500 * 1024;
|
|
1120
|
-
async function resolveEntryFiles(packageDir) {
|
|
1121
|
-
if (!existsSync(join(packageDir, "package.json"))) return [];
|
|
1122
|
-
const files = await glob(["**/*.d.{ts,mts,cts}"], {
|
|
1123
|
-
cwd: packageDir,
|
|
1124
|
-
ignore: [...SKIP_DIRS.map((d) => `**/${d}/**`), ...SKIP_PATTERNS],
|
|
1125
|
-
absolute: false,
|
|
1126
|
-
expandDirectories: false
|
|
1127
|
-
});
|
|
1128
|
-
const entries = [];
|
|
1129
|
-
for (const file of files) {
|
|
1130
|
-
const absPath = join(packageDir, file);
|
|
1131
|
-
let content;
|
|
1132
|
-
try {
|
|
1133
|
-
content = readFileSync(absPath, "utf-8");
|
|
1134
|
-
} catch {
|
|
1135
|
-
continue;
|
|
902
|
+
async function findGitTag(owner, repo, version, packageName, branchHint) {
|
|
903
|
+
const candidates = [`v${version}`, version];
|
|
904
|
+
if (packageName) candidates.push(`${packageName}@${version}`);
|
|
905
|
+
for (const tag of candidates) {
|
|
906
|
+
const files = await listFilesAtRef(owner, repo, tag);
|
|
907
|
+
if (files.length > 0) return {
|
|
908
|
+
ref: tag,
|
|
909
|
+
files
|
|
910
|
+
};
|
|
911
|
+
}
|
|
912
|
+
if (packageName) {
|
|
913
|
+
const latestTag = await findLatestReleaseTag(owner, repo, packageName);
|
|
914
|
+
if (latestTag) {
|
|
915
|
+
const files = await listFilesAtRef(owner, repo, latestTag);
|
|
916
|
+
if (files.length > 0) return {
|
|
917
|
+
ref: latestTag,
|
|
918
|
+
files
|
|
919
|
+
};
|
|
1136
920
|
}
|
|
1137
|
-
if (content.length > MAX_FILE_SIZE) continue;
|
|
1138
|
-
entries.push({
|
|
1139
|
-
path: file,
|
|
1140
|
-
content,
|
|
1141
|
-
type: "types"
|
|
1142
|
-
});
|
|
1143
921
|
}
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
localPath: trimmed.startsWith("~") ? resolve(process.env.HOME || "", trimmed.slice(1)) : resolve(trimmed)
|
|
1152
|
-
};
|
|
1153
|
-
if (trimmed.startsWith("git@")) {
|
|
1154
|
-
const gh = parseGitHubUrl(normalizeRepoUrl(trimmed));
|
|
1155
|
-
if (gh) return {
|
|
1156
|
-
type: "github",
|
|
1157
|
-
owner: gh.owner,
|
|
1158
|
-
repo: gh.repo
|
|
922
|
+
const branches = branchHint ? [branchHint, ...["main", "master"].filter((b) => b !== branchHint)] : ["main", "master"];
|
|
923
|
+
for (const branch of branches) {
|
|
924
|
+
const files = await listFilesAtRef(owner, repo, branch);
|
|
925
|
+
if (files.length > 0) return {
|
|
926
|
+
ref: branch,
|
|
927
|
+
files,
|
|
928
|
+
fallback: true
|
|
1159
929
|
};
|
|
1160
|
-
return null;
|
|
1161
930
|
}
|
|
1162
|
-
if (trimmed.startsWith("https://") || trimmed.startsWith("http://")) return parseGitUrl(trimmed);
|
|
1163
|
-
if (/^[\w.-]+\/[\w.-]+$/.test(trimmed)) return {
|
|
1164
|
-
type: "github",
|
|
1165
|
-
owner: trimmed.split("/")[0],
|
|
1166
|
-
repo: trimmed.split("/")[1]
|
|
1167
|
-
};
|
|
1168
931
|
return null;
|
|
1169
932
|
}
|
|
1170
|
-
function
|
|
1171
|
-
|
|
1172
|
-
const
|
|
1173
|
-
if (
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
ref: parts[3],
|
|
1183
|
-
skillPath: parts.length > 4 ? parts.slice(4).join("/") : void 0
|
|
1184
|
-
};
|
|
1185
|
-
return {
|
|
1186
|
-
type: "github",
|
|
1187
|
-
owner,
|
|
1188
|
-
repo
|
|
1189
|
-
};
|
|
1190
|
-
}
|
|
1191
|
-
if (parsed.hostname === "gitlab.com") {
|
|
1192
|
-
const parts = parsed.pathname.replace(/^\//, "").replace(/\.git$/, "").split("/");
|
|
1193
|
-
const owner = parts[0];
|
|
1194
|
-
const repo = parts[1];
|
|
1195
|
-
if (!owner || !repo) return null;
|
|
1196
|
-
return {
|
|
1197
|
-
type: "gitlab",
|
|
1198
|
-
owner,
|
|
1199
|
-
repo
|
|
1200
|
-
};
|
|
1201
|
-
}
|
|
1202
|
-
return null;
|
|
1203
|
-
} catch {
|
|
1204
|
-
return null;
|
|
933
|
+
async function fetchUnghReleases(owner, repo) {
|
|
934
|
+
if (!isKnownPrivateRepo(owner, repo)) {
|
|
935
|
+
const data = await $fetch(`https://ungh.cc/repos/${owner}/${repo}/releases`).catch(() => null);
|
|
936
|
+
if (data?.releases?.length) return data.releases;
|
|
937
|
+
}
|
|
938
|
+
const raw = await ghApiPaginated(`repos/${owner}/${repo}/releases`);
|
|
939
|
+
if (raw.length > 0) {
|
|
940
|
+
markRepoPrivate(owner, repo);
|
|
941
|
+
return raw.map((r) => ({
|
|
942
|
+
tag: r.tag_name,
|
|
943
|
+
publishedAt: r.published_at
|
|
944
|
+
}));
|
|
1205
945
|
}
|
|
946
|
+
return [];
|
|
1206
947
|
}
|
|
1207
|
-
function
|
|
1208
|
-
const
|
|
1209
|
-
return
|
|
1210
|
-
name: fm.name,
|
|
1211
|
-
description: fm.description
|
|
1212
|
-
};
|
|
948
|
+
async function findLatestReleaseTag(owner, repo, packageName) {
|
|
949
|
+
const prefix = `${packageName}@`;
|
|
950
|
+
return (await fetchUnghReleases(owner, repo)).find((r) => r.tag.startsWith(prefix))?.tag ?? null;
|
|
1213
951
|
}
|
|
1214
|
-
function
|
|
1215
|
-
|
|
1216
|
-
if (!existsSync(dir)) return files;
|
|
1217
|
-
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
1218
|
-
const relPath = prefix ? `${prefix}/${entry.name}` : entry.name;
|
|
1219
|
-
const fullPath = resolve(dir, entry.name);
|
|
1220
|
-
if (entry.isDirectory()) files.push(...collectFiles(fullPath, relPath));
|
|
1221
|
-
else if (entry.isFile()) files.push({
|
|
1222
|
-
path: relPath,
|
|
1223
|
-
content: readFileSync(fullPath, "utf-8")
|
|
1224
|
-
});
|
|
1225
|
-
}
|
|
1226
|
-
return files;
|
|
952
|
+
function filterDocFiles(files, pathPrefix) {
|
|
953
|
+
return files.filter((f) => f.startsWith(pathPrefix) && /\.(?:md|mdx)$/.test(f));
|
|
1227
954
|
}
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
955
|
+
const FRAMEWORK_NAMES = new Set([
|
|
956
|
+
"vue",
|
|
957
|
+
"react",
|
|
958
|
+
"solid",
|
|
959
|
+
"angular",
|
|
960
|
+
"svelte",
|
|
961
|
+
"preact",
|
|
962
|
+
"lit",
|
|
963
|
+
"qwik"
|
|
964
|
+
]);
|
|
965
|
+
function filterFrameworkDocs(files, packageName) {
|
|
966
|
+
if (!packageName) return files;
|
|
967
|
+
const shortName = packageName.replace(/^@.*\//, "");
|
|
968
|
+
const targetFramework = [...FRAMEWORK_NAMES].find((fw) => shortName.includes(fw));
|
|
969
|
+
if (!targetFramework) return files;
|
|
970
|
+
const otherFrameworks = [...FRAMEWORK_NAMES].filter((fw) => fw !== targetFramework);
|
|
971
|
+
const excludePattern = new RegExp(`\\b(?:${otherFrameworks.join("|")})\\b`);
|
|
972
|
+
return files.filter((f) => !excludePattern.test(f));
|
|
1233
973
|
}
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
974
|
+
const NOISE_PATTERNS = [
|
|
975
|
+
/^\.changeset\//,
|
|
976
|
+
/CHANGELOG\.md$/i,
|
|
977
|
+
/CONTRIBUTING\.md$/i,
|
|
978
|
+
/^\.github\//
|
|
979
|
+
];
|
|
980
|
+
const EXCLUDE_DIRS = new Set([
|
|
981
|
+
"test",
|
|
982
|
+
"tests",
|
|
983
|
+
"__tests__",
|
|
984
|
+
"fixtures",
|
|
985
|
+
"fixture",
|
|
986
|
+
"examples",
|
|
987
|
+
"example",
|
|
988
|
+
"node_modules",
|
|
989
|
+
".git",
|
|
990
|
+
"dist",
|
|
991
|
+
"build",
|
|
992
|
+
"coverage",
|
|
993
|
+
"e2e",
|
|
994
|
+
"spec",
|
|
995
|
+
"mocks",
|
|
996
|
+
"__mocks__"
|
|
997
|
+
]);
|
|
998
|
+
const DOC_DIR_BONUS = new Set([
|
|
999
|
+
"docs",
|
|
1000
|
+
"documentation",
|
|
1001
|
+
"pages",
|
|
1002
|
+
"content",
|
|
1003
|
+
"website",
|
|
1004
|
+
"guide",
|
|
1005
|
+
"guides",
|
|
1006
|
+
"wiki",
|
|
1007
|
+
"manual",
|
|
1008
|
+
"api"
|
|
1009
|
+
]);
|
|
1010
|
+
function hasExcludedDir(path) {
|
|
1011
|
+
return path.split("/").some((p) => EXCLUDE_DIRS.has(p.toLowerCase()));
|
|
1012
|
+
}
|
|
1013
|
+
function getPathDepth(path) {
|
|
1014
|
+
return path.split("/").filter(Boolean).length;
|
|
1015
|
+
}
|
|
1016
|
+
function hasDocDirBonus(path) {
|
|
1017
|
+
return path.split("/").some((p) => DOC_DIR_BONUS.has(p.toLowerCase()));
|
|
1018
|
+
}
|
|
1019
|
+
function scoreDocDir(dir, fileCount) {
|
|
1020
|
+
const depth = getPathDepth(dir) || 1;
|
|
1021
|
+
return fileCount * (hasDocDirBonus(dir) ? 1.5 : 1) / depth;
|
|
1022
|
+
}
|
|
1023
|
+
function discoverDocFiles(allFiles, packageName) {
|
|
1024
|
+
const mdFiles = allFiles.filter((f) => /\.(?:md|mdx)$/.test(f)).filter((f) => !NOISE_PATTERNS.some((p) => p.test(f))).filter((f) => f.includes("/"));
|
|
1025
|
+
if (packageName?.includes("/")) {
|
|
1026
|
+
const subPkgPrefix = `packages/${packageName.split("/").pop().toLowerCase()}/`;
|
|
1027
|
+
const subPkgFiles = mdFiles.filter((f) => f.startsWith(subPkgPrefix));
|
|
1028
|
+
if (subPkgFiles.length >= 3) return {
|
|
1029
|
+
files: subPkgFiles,
|
|
1030
|
+
prefix: subPkgPrefix
|
|
1031
|
+
};
|
|
1243
1032
|
}
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1033
|
+
const docsGroups = /* @__PURE__ */ new Map();
|
|
1034
|
+
for (const file of mdFiles) {
|
|
1035
|
+
const docsIdx = file.lastIndexOf("/docs/");
|
|
1036
|
+
if (docsIdx === -1) continue;
|
|
1037
|
+
mapInsert(docsGroups, file.slice(0, docsIdx + 6), () => []).push(file);
|
|
1247
1038
|
}
|
|
1248
|
-
|
|
1039
|
+
if (docsGroups.size > 0) {
|
|
1040
|
+
const largest = [...docsGroups.entries()].sort((a, b) => b[1].length - a[1].length)[0];
|
|
1041
|
+
if (largest[1].length >= 3) {
|
|
1042
|
+
const fullPrefix = largest[0];
|
|
1043
|
+
const docsIdx = fullPrefix.lastIndexOf("docs/");
|
|
1044
|
+
const stripPrefix = docsIdx > 0 ? fullPrefix.slice(0, docsIdx) : "";
|
|
1045
|
+
return {
|
|
1046
|
+
files: largest[1],
|
|
1047
|
+
prefix: stripPrefix
|
|
1048
|
+
};
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
const dirGroups = /* @__PURE__ */ new Map();
|
|
1052
|
+
for (const file of mdFiles) {
|
|
1053
|
+
if (hasExcludedDir(file)) continue;
|
|
1054
|
+
const lastSlash = file.lastIndexOf("/");
|
|
1055
|
+
if (lastSlash === -1) continue;
|
|
1056
|
+
mapInsert(dirGroups, file.slice(0, lastSlash + 1), () => []).push(file);
|
|
1057
|
+
}
|
|
1058
|
+
if (dirGroups.size === 0) return null;
|
|
1059
|
+
const scored = Array.from(dirGroups.entries(), ([dir, files]) => ({
|
|
1060
|
+
dir,
|
|
1061
|
+
files,
|
|
1062
|
+
score: scoreDocDir(dir, files.length)
|
|
1063
|
+
})).filter((d) => d.files.length >= 5).sort((a, b) => b.score - a.score);
|
|
1064
|
+
if (scored.length === 0) return null;
|
|
1065
|
+
const best = scored[0];
|
|
1066
|
+
return {
|
|
1067
|
+
files: best.files,
|
|
1068
|
+
prefix: best.dir
|
|
1069
|
+
};
|
|
1249
1070
|
}
|
|
1250
|
-
function
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
const
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1071
|
+
async function listDocsAtRef(owner, repo, ref, pathPrefix = "docs/") {
|
|
1072
|
+
return filterDocFiles(await listFilesAtRef(owner, repo, ref), pathPrefix);
|
|
1073
|
+
}
|
|
1074
|
+
async function fetchGitDocs(owner, repo, version, packageName, repoUrl) {
|
|
1075
|
+
const override = packageName ? getDocOverride(packageName) : void 0;
|
|
1076
|
+
if (override) {
|
|
1077
|
+
const ref = override.ref || "main";
|
|
1078
|
+
const fallback = !override.ref;
|
|
1079
|
+
const files = await listDocsAtRef(override.owner, override.repo, ref, `${override.path}/`);
|
|
1080
|
+
if (files.length === 0) return null;
|
|
1081
|
+
return {
|
|
1082
|
+
baseUrl: `https://raw.githubusercontent.com/${override.owner}/${override.repo}/${ref}`,
|
|
1083
|
+
ref,
|
|
1084
|
+
files,
|
|
1085
|
+
fallback,
|
|
1086
|
+
docsPrefix: `${override.path}/` !== "docs/" ? `${override.path}/` : void 0
|
|
1087
|
+
};
|
|
1088
|
+
}
|
|
1089
|
+
const tag = await findGitTag(owner, repo, version, packageName, repoUrl ? extractBranchHint(repoUrl) : void 0);
|
|
1090
|
+
if (!tag) return null;
|
|
1091
|
+
let docs = filterDocFiles(tag.files, "docs/");
|
|
1092
|
+
let docsPrefix;
|
|
1093
|
+
let allFiles;
|
|
1094
|
+
if (docs.length === 0) {
|
|
1095
|
+
const discovered = discoverDocFiles(tag.files, packageName);
|
|
1096
|
+
if (discovered) {
|
|
1097
|
+
docs = discovered.files;
|
|
1098
|
+
docsPrefix = discovered.prefix || void 0;
|
|
1099
|
+
allFiles = tag.files;
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1102
|
+
docs = filterFrameworkDocs(docs, packageName);
|
|
1103
|
+
if (docs.length === 0) return null;
|
|
1258
1104
|
return {
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1105
|
+
baseUrl: `https://raw.githubusercontent.com/${owner}/${repo}/${tag.ref}`,
|
|
1106
|
+
ref: tag.ref,
|
|
1107
|
+
files: docs,
|
|
1108
|
+
docsPrefix,
|
|
1109
|
+
allFiles,
|
|
1110
|
+
fallback: tag.fallback
|
|
1264
1111
|
};
|
|
1265
1112
|
}
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1113
|
+
function normalizePath(p) {
|
|
1114
|
+
return p.replace(/^\//, "").replace(/\.(?:md|mdx)$/, "");
|
|
1115
|
+
}
|
|
1116
|
+
function validateGitDocsWithLlms(llmsLinks, repoFiles) {
|
|
1117
|
+
if (llmsLinks.length === 0) return {
|
|
1118
|
+
isValid: true,
|
|
1119
|
+
matchRatio: 1
|
|
1120
|
+
};
|
|
1121
|
+
const sample = llmsLinks.slice(0, 10);
|
|
1122
|
+
const normalizedLinks = sample.map((link) => {
|
|
1123
|
+
let path = link.url;
|
|
1124
|
+
if (path.startsWith("http")) try {
|
|
1125
|
+
path = new URL(path).pathname;
|
|
1126
|
+
} catch {}
|
|
1127
|
+
return normalizePath(path);
|
|
1128
|
+
});
|
|
1129
|
+
const repoNormalized = new Set(repoFiles.map(normalizePath));
|
|
1130
|
+
let matches = 0;
|
|
1131
|
+
for (const linkPath of normalizedLinks) for (const repoPath of repoNormalized) if (repoPath === linkPath || repoPath.endsWith(`/${linkPath}`)) {
|
|
1132
|
+
matches++;
|
|
1133
|
+
break;
|
|
1274
1134
|
}
|
|
1275
|
-
|
|
1135
|
+
const matchRatio = matches / sample.length;
|
|
1136
|
+
return {
|
|
1137
|
+
isValid: matchRatio >= .3,
|
|
1138
|
+
matchRatio
|
|
1139
|
+
};
|
|
1276
1140
|
}
|
|
1277
|
-
async function
|
|
1278
|
-
const
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
const skill = readLocalSkill(dir, skillPath);
|
|
1288
|
-
return skill ? [skill] : [];
|
|
1289
|
-
}
|
|
1290
|
-
onProgress?.(`Downloading ${owner}/${repo}/skills@${ref}`);
|
|
1141
|
+
async function verifyNpmRepo(owner, repo, packageName) {
|
|
1142
|
+
const base = `https://raw.githubusercontent.com/${owner}/${repo}/HEAD`;
|
|
1143
|
+
const paths = [
|
|
1144
|
+
"package.json",
|
|
1145
|
+
`packages/${packageName.replace(/^@.*\//, "")}/package.json`,
|
|
1146
|
+
`packages/${packageName.replace(/^@/, "").replace("/", "-")}/package.json`
|
|
1147
|
+
];
|
|
1148
|
+
for (const path of paths) {
|
|
1149
|
+
const text = await fetchGitHubRaw(`${base}/${path}`);
|
|
1150
|
+
if (!text) continue;
|
|
1291
1151
|
try {
|
|
1292
|
-
|
|
1293
|
-
dir: tempDir,
|
|
1294
|
-
force: true,
|
|
1295
|
-
auth: getGitHubToken() || void 0
|
|
1296
|
-
});
|
|
1297
|
-
const skills = [];
|
|
1298
|
-
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
1299
|
-
if (!entry.isDirectory()) continue;
|
|
1300
|
-
const skill = readLocalSkill(resolve(dir, entry.name), `skills/${entry.name}`);
|
|
1301
|
-
if (skill) skills.push(skill);
|
|
1302
|
-
}
|
|
1303
|
-
if (skills.length > 0) {
|
|
1304
|
-
onProgress?.(`Found ${skills.length} skill(s)`);
|
|
1305
|
-
return skills;
|
|
1306
|
-
}
|
|
1152
|
+
if (JSON.parse(text).name === packageName) return true;
|
|
1307
1153
|
} catch {}
|
|
1308
|
-
const content = await fetchGitHubRaw(`https://raw.githubusercontent.com/${owner}/${repo}/${ref}/SKILL.md`);
|
|
1309
|
-
if (content) {
|
|
1310
|
-
const fm = parseSkillFrontmatterName(content);
|
|
1311
|
-
onProgress?.("Found 1 skill");
|
|
1312
|
-
return [{
|
|
1313
|
-
name: fm.name || repo,
|
|
1314
|
-
description: fm.description || "",
|
|
1315
|
-
path: "",
|
|
1316
|
-
content,
|
|
1317
|
-
files: []
|
|
1318
|
-
}];
|
|
1319
|
-
}
|
|
1320
|
-
return [];
|
|
1321
|
-
} catch {
|
|
1322
|
-
return [];
|
|
1323
|
-
} finally {
|
|
1324
|
-
rmSync(tempDir, {
|
|
1325
|
-
recursive: true,
|
|
1326
|
-
force: true
|
|
1327
|
-
});
|
|
1328
1154
|
}
|
|
1155
|
+
return false;
|
|
1329
1156
|
}
|
|
1330
|
-
async function
|
|
1331
|
-
const
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
const subdir = source.skillPath || "skills";
|
|
1337
|
-
onProgress?.(`Downloading ${owner}/${repo}/${subdir}@${ref}`);
|
|
1338
|
-
const { dir } = await downloadTemplate(`gitlab:${owner}/${repo}/${subdir}#${ref}`, {
|
|
1339
|
-
dir: tempDir,
|
|
1340
|
-
force: true
|
|
1341
|
-
});
|
|
1342
|
-
if (source.skillPath) {
|
|
1343
|
-
const skill = readLocalSkill(dir, source.skillPath);
|
|
1344
|
-
return { skills: skill ? [skill] : [] };
|
|
1345
|
-
}
|
|
1346
|
-
const skills = [];
|
|
1347
|
-
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
1348
|
-
if (!entry.isDirectory()) continue;
|
|
1349
|
-
const skill = readLocalSkill(resolve(dir, entry.name), `skills/${entry.name}`);
|
|
1350
|
-
if (skill) skills.push(skill);
|
|
1351
|
-
}
|
|
1352
|
-
if (skills.length > 0) {
|
|
1353
|
-
onProgress?.(`Found ${skills.length} skill(s)`);
|
|
1354
|
-
return { skills };
|
|
1355
|
-
}
|
|
1356
|
-
const content = await $fetch(`https://gitlab.com/${owner}/${repo}/-/raw/${ref}/SKILL.md`, { responseType: "text" }).catch(() => null);
|
|
1357
|
-
if (content) {
|
|
1358
|
-
const fm = parseSkillFrontmatterName(content);
|
|
1359
|
-
return { skills: [{
|
|
1360
|
-
name: fm.name || repo,
|
|
1361
|
-
description: fm.description || "",
|
|
1362
|
-
path: "",
|
|
1363
|
-
content,
|
|
1364
|
-
files: []
|
|
1365
|
-
}] };
|
|
1157
|
+
async function searchGitHubRepo(packageName) {
|
|
1158
|
+
const shortName = packageName.replace(/^@.*\//, "");
|
|
1159
|
+
for (const candidate of [packageName.replace(/^@/, "").replace("/", "/"), shortName]) {
|
|
1160
|
+
if (!candidate.includes("/")) {
|
|
1161
|
+
if ((await $fetch.raw(`https://ungh.cc/repos/${shortName}/${shortName}`).catch(() => null))?.ok) return `https://github.com/${shortName}/${shortName}`;
|
|
1162
|
+
continue;
|
|
1366
1163
|
}
|
|
1367
|
-
|
|
1368
|
-
}
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1164
|
+
if ((await $fetch.raw(`https://ungh.cc/repos/${candidate}`).catch(() => null))?.ok) return `https://github.com/${candidate}`;
|
|
1165
|
+
}
|
|
1166
|
+
const searchTerm = packageName.replace(/^@/, "");
|
|
1167
|
+
if (isGhAvailable()) try {
|
|
1168
|
+
const { stdout: json } = spawnSync("gh", [
|
|
1169
|
+
"search",
|
|
1170
|
+
"repos",
|
|
1171
|
+
searchTerm,
|
|
1172
|
+
"--json",
|
|
1173
|
+
"fullName",
|
|
1174
|
+
"--limit",
|
|
1175
|
+
"5"
|
|
1176
|
+
], {
|
|
1177
|
+
encoding: "utf-8",
|
|
1178
|
+
timeout: 15e3
|
|
1374
1179
|
});
|
|
1180
|
+
if (!json) throw new Error("no output");
|
|
1181
|
+
const repos = JSON.parse(json);
|
|
1182
|
+
const match = repos.find((r) => r.fullName.toLowerCase().endsWith(`/${packageName.toLowerCase()}`) || r.fullName.toLowerCase().endsWith(`/${shortName.toLowerCase()}`));
|
|
1183
|
+
if (match) return `https://github.com/${match.fullName}`;
|
|
1184
|
+
for (const candidate of repos) {
|
|
1185
|
+
const gh = parseGitHubUrl(`https://github.com/${candidate.fullName}`);
|
|
1186
|
+
if (gh && await verifyNpmRepo(gh.owner, gh.repo, packageName)) return `https://github.com/${candidate.fullName}`;
|
|
1187
|
+
}
|
|
1188
|
+
} catch {}
|
|
1189
|
+
const data = await $fetch(`https://api.github.com/search/repositories?q=${encodeURIComponent(`${searchTerm} in:name`)}&per_page=5`).catch(() => null);
|
|
1190
|
+
if (!data?.items?.length) return null;
|
|
1191
|
+
const match = data.items.find((r) => r.full_name.toLowerCase().endsWith(`/${packageName.toLowerCase()}`) || r.full_name.toLowerCase().endsWith(`/${shortName.toLowerCase()}`));
|
|
1192
|
+
if (match) return `https://github.com/${match.full_name}`;
|
|
1193
|
+
for (const candidate of data.items) {
|
|
1194
|
+
const gh = parseGitHubUrl(`https://github.com/${candidate.full_name}`);
|
|
1195
|
+
if (gh && await verifyNpmRepo(gh.owner, gh.repo, packageName)) return `https://github.com/${candidate.full_name}`;
|
|
1375
1196
|
}
|
|
1376
|
-
}
|
|
1377
|
-
async function fetchLlmsUrl(docsUrl) {
|
|
1378
|
-
const llmsUrl = `${new URL(docsUrl).origin}/llms.txt`;
|
|
1379
|
-
if (await verifyUrl(llmsUrl)) return llmsUrl;
|
|
1380
1197
|
return null;
|
|
1381
1198
|
}
|
|
1382
|
-
async function
|
|
1383
|
-
const
|
|
1384
|
-
if (
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
links: parseMarkdownLinks(content)
|
|
1388
|
-
};
|
|
1389
|
-
}
|
|
1390
|
-
function parseMarkdownLinks(content) {
|
|
1391
|
-
return extractLinks(content).filter((l) => l.url.endsWith(".md"));
|
|
1392
|
-
}
|
|
1393
|
-
function isSafeUrl(url) {
|
|
1394
|
-
try {
|
|
1395
|
-
const parsed = new URL(url);
|
|
1396
|
-
if (parsed.protocol !== "https:") return false;
|
|
1397
|
-
const host = parsed.hostname;
|
|
1398
|
-
if (host === "localhost" || host === "0.0.0.0" || host === "[::1]") return false;
|
|
1399
|
-
if (/^(?:127\.|10\.|172\.(?:1[6-9]|2\d|3[01])\.|192\.168\.|169\.254\.)/.test(host)) return false;
|
|
1400
|
-
if (/^\[(?:f[cd]|fe[89ab]|::ffff:)/i.test(host)) return false;
|
|
1401
|
-
return true;
|
|
1402
|
-
} catch {
|
|
1403
|
-
return false;
|
|
1404
|
-
}
|
|
1405
|
-
}
|
|
1406
|
-
async function downloadLlmsDocs(llmsContent, baseUrl, onProgress) {
|
|
1407
|
-
const limit = pLimit(5);
|
|
1408
|
-
let completed = 0;
|
|
1409
|
-
return (await Promise.all(llmsContent.links.map((link) => limit(async () => {
|
|
1410
|
-
const url = link.url.startsWith("http") ? link.url : `${baseUrl.replace(/\/$/, "")}${link.url.startsWith("/") ? "" : "/"}${link.url}`;
|
|
1411
|
-
if (!isSafeUrl(url)) return null;
|
|
1412
|
-
const content = await fetchText(url);
|
|
1413
|
-
onProgress?.(link.url, ++completed, llmsContent.links.length);
|
|
1414
|
-
if (content && content.length > 100) return {
|
|
1415
|
-
url: link.url.startsWith("http") ? new URL(link.url).pathname : link.url,
|
|
1416
|
-
title: link.title,
|
|
1417
|
-
content
|
|
1418
|
-
};
|
|
1419
|
-
return null;
|
|
1420
|
-
})))).filter((d) => d !== null);
|
|
1421
|
-
}
|
|
1422
|
-
function normalizeLlmsLinks(content, baseUrl) {
|
|
1423
|
-
let normalized = content;
|
|
1424
|
-
if (baseUrl) {
|
|
1425
|
-
const escaped = baseUrl.replace(/\/$/, "").replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
1426
|
-
normalized = normalized.replace(new RegExp(`\\]\\(${escaped}(/[^)]+\\.md)\\)`, "g"), "](./docs$1)");
|
|
1427
|
-
}
|
|
1428
|
-
normalized = normalized.replace(/\]\(\/([^)]+\.md)\)/g, "](./docs/$1)");
|
|
1429
|
-
return normalized;
|
|
1430
|
-
}
|
|
1431
|
-
function extractSections(content, patterns) {
|
|
1432
|
-
const sections = [];
|
|
1433
|
-
const parts = content.split(/\n---\n/);
|
|
1434
|
-
for (const part of parts) {
|
|
1435
|
-
const urlMatch = part.match(/^url: *(\S.*)$/m);
|
|
1436
|
-
if (!urlMatch) continue;
|
|
1437
|
-
const url = urlMatch[1];
|
|
1438
|
-
if (patterns.some((p) => url.includes(p))) {
|
|
1439
|
-
const contentStart = part.indexOf("\n", part.indexOf("url:"));
|
|
1440
|
-
if (contentStart > -1) sections.push(part.slice(contentStart + 1));
|
|
1441
|
-
}
|
|
1442
|
-
}
|
|
1443
|
-
if (sections.length === 0) return null;
|
|
1444
|
-
return sections.join("\n\n---\n\n");
|
|
1199
|
+
async function fetchGitHubRepoMeta(owner, repo, packageName) {
|
|
1200
|
+
const override = packageName ? getDocOverride(packageName) : void 0;
|
|
1201
|
+
if (override?.homepage) return { homepage: override.homepage };
|
|
1202
|
+
const data = await ghApi(`repos/${owner}/${repo}`) ?? await $fetch(`https://api.github.com/repos/${owner}/${repo}`).catch(() => null);
|
|
1203
|
+
return data?.homepage ? { homepage: data.homepage } : null;
|
|
1445
1204
|
}
|
|
1446
|
-
|
|
1447
|
-
const
|
|
1448
|
-
async function listFilesAtRef(owner, repo, ref) {
|
|
1205
|
+
async function fetchReadme(owner, repo, subdir, ref) {
|
|
1206
|
+
const branch = ref || "main";
|
|
1449
1207
|
if (!isKnownPrivateRepo(owner, repo)) {
|
|
1450
|
-
const
|
|
1451
|
-
if (
|
|
1208
|
+
const unghUrl = subdir ? `https://ungh.cc/repos/${owner}/${repo}/files/${branch}/${subdir}/README.md` : `https://ungh.cc/repos/${owner}/${repo}/readme${ref ? `?ref=${ref}` : ""}`;
|
|
1209
|
+
if ((await $fetch.raw(unghUrl).catch(() => null))?.ok) return `ungh://${owner}/${repo}${subdir ? `/${subdir}` : ""}${ref ? `@${ref}` : ""}`;
|
|
1452
1210
|
}
|
|
1453
|
-
const
|
|
1454
|
-
|
|
1211
|
+
const basePath = subdir ? `${subdir}/` : "";
|
|
1212
|
+
const branches = ref ? [ref] : ["main", "master"];
|
|
1213
|
+
const token = isKnownPrivateRepo(owner, repo) ? getGitHubToken() : null;
|
|
1214
|
+
const authHeaders = token ? { Authorization: `token ${token}` } : {};
|
|
1215
|
+
for (const b of branches) for (const filename of [
|
|
1216
|
+
"README.md",
|
|
1217
|
+
"Readme.md",
|
|
1218
|
+
"readme.md"
|
|
1219
|
+
]) {
|
|
1220
|
+
const readmeUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${b}/${basePath}${filename}`;
|
|
1221
|
+
if ((await $fetch.raw(readmeUrl, { headers: authHeaders }).catch(() => null))?.ok) return readmeUrl;
|
|
1222
|
+
}
|
|
1223
|
+
const refParam = ref ? `?ref=${ref}` : "";
|
|
1224
|
+
const apiData = await ghApi(subdir ? `repos/${owner}/${repo}/contents/${subdir}/README.md${refParam}` : `repos/${owner}/${repo}/readme${refParam}`);
|
|
1225
|
+
if (apiData?.download_url) {
|
|
1455
1226
|
markRepoPrivate(owner, repo);
|
|
1456
|
-
return
|
|
1227
|
+
return apiData.download_url;
|
|
1457
1228
|
}
|
|
1458
|
-
return
|
|
1229
|
+
return null;
|
|
1459
1230
|
}
|
|
1460
|
-
async function
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
if (files.length > 0) return {
|
|
1466
|
-
ref: tag,
|
|
1467
|
-
files
|
|
1468
|
-
};
|
|
1231
|
+
async function fetchReadmeContent(url) {
|
|
1232
|
+
if (url.startsWith("file://")) {
|
|
1233
|
+
const filePath = fileURLToPath(url);
|
|
1234
|
+
if (!existsSync(filePath)) return null;
|
|
1235
|
+
return readFileSync(filePath, "utf-8");
|
|
1469
1236
|
}
|
|
1470
|
-
if (
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1237
|
+
if (url.startsWith("ungh://")) {
|
|
1238
|
+
let path = url.replace("ungh://", "");
|
|
1239
|
+
let ref = "main";
|
|
1240
|
+
const atIdx = path.lastIndexOf("@");
|
|
1241
|
+
if (atIdx !== -1) {
|
|
1242
|
+
ref = path.slice(atIdx + 1);
|
|
1243
|
+
path = path.slice(0, atIdx);
|
|
1244
|
+
}
|
|
1245
|
+
const parts = path.split("/");
|
|
1246
|
+
const owner = parts[0];
|
|
1247
|
+
const repo = parts[1];
|
|
1248
|
+
const subdir = parts.slice(2).join("/");
|
|
1249
|
+
const text = await $fetch(subdir ? `https://ungh.cc/repos/${owner}/${repo}/files/${ref}/${subdir}/README.md` : `https://ungh.cc/repos/${owner}/${repo}/readme?ref=${ref}`, { responseType: "text" }).catch(() => null);
|
|
1250
|
+
if (!text) return null;
|
|
1251
|
+
try {
|
|
1252
|
+
const json = JSON.parse(text);
|
|
1253
|
+
return json.markdown || json.file?.contents || null;
|
|
1254
|
+
} catch {
|
|
1255
|
+
return text;
|
|
1478
1256
|
}
|
|
1479
1257
|
}
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1258
|
+
if (url.includes("raw.githubusercontent.com")) return fetchGitHubRaw(url);
|
|
1259
|
+
return fetchText(url);
|
|
1260
|
+
}
|
|
1261
|
+
async function resolveGitHubRepo(owner, repo, onProgress) {
|
|
1262
|
+
onProgress?.("Fetching repo metadata");
|
|
1263
|
+
const repoUrl = `https://github.com/${owner}/${repo}`;
|
|
1264
|
+
const meta = await ghApi(`repos/${owner}/${repo}`) ?? await $fetch(`https://api.github.com/repos/${owner}/${repo}`).catch(() => null);
|
|
1265
|
+
const homepage = meta?.homepage || void 0;
|
|
1266
|
+
const description = meta?.description || void 0;
|
|
1267
|
+
onProgress?.("Fetching latest release");
|
|
1268
|
+
const releases = await fetchUnghReleases(owner, repo);
|
|
1269
|
+
let version = "main";
|
|
1270
|
+
let releasedAt;
|
|
1271
|
+
const latestRelease = releases[0];
|
|
1272
|
+
if (latestRelease) {
|
|
1273
|
+
version = latestRelease.tag.replace(/^v/, "");
|
|
1274
|
+
releasedAt = latestRelease.publishedAt;
|
|
1275
|
+
}
|
|
1276
|
+
onProgress?.("Resolving docs");
|
|
1277
|
+
const gitDocs = await fetchGitDocs(owner, repo, version);
|
|
1278
|
+
const gitDocsUrl = gitDocs ? `${repoUrl}/tree/${gitDocs.ref}/docs` : void 0;
|
|
1279
|
+
const gitRef = gitDocs?.ref;
|
|
1280
|
+
onProgress?.("Fetching README");
|
|
1281
|
+
const readmeUrl = await fetchReadme(owner, repo);
|
|
1282
|
+
let llmsUrl;
|
|
1283
|
+
if (homepage) {
|
|
1284
|
+
onProgress?.("Checking llms.txt");
|
|
1285
|
+
llmsUrl = await fetchLlmsUrl(homepage).catch(() => null) ?? void 0;
|
|
1286
|
+
}
|
|
1287
|
+
if (!gitDocsUrl && !readmeUrl && !llmsUrl) return null;
|
|
1288
|
+
return {
|
|
1289
|
+
name: repo,
|
|
1290
|
+
version: latestRelease ? version : void 0,
|
|
1291
|
+
releasedAt,
|
|
1292
|
+
description,
|
|
1293
|
+
repoUrl,
|
|
1294
|
+
docsUrl: homepage,
|
|
1295
|
+
gitDocsUrl,
|
|
1296
|
+
gitRef,
|
|
1297
|
+
gitDocsFallback: gitDocs?.fallback,
|
|
1298
|
+
readmeUrl: readmeUrl ?? void 0,
|
|
1299
|
+
llmsUrl
|
|
1300
|
+
};
|
|
1301
|
+
}
|
|
1302
|
+
const VALID_CRATE_NAME = /^[a-z0-9][\w-]*$/;
|
|
1303
|
+
const runCratesApiRateLimited = createRateLimitedRunner(1e3);
|
|
1304
|
+
function selectCrateVersion(data, requestedVersion) {
|
|
1305
|
+
const versions = data.versions || [];
|
|
1306
|
+
if (requestedVersion) {
|
|
1307
|
+
const exact = versions.find((v) => v.num === requestedVersion && !v.yanked);
|
|
1308
|
+
if (exact?.num) return {
|
|
1309
|
+
version: exact.num,
|
|
1310
|
+
entry: exact
|
|
1487
1311
|
};
|
|
1488
1312
|
}
|
|
1313
|
+
const crate = data.crate;
|
|
1314
|
+
const preferred = [
|
|
1315
|
+
crate?.max_stable_version,
|
|
1316
|
+
crate?.newest_version,
|
|
1317
|
+
crate?.max_version,
|
|
1318
|
+
crate?.default_version
|
|
1319
|
+
].find(Boolean);
|
|
1320
|
+
if (preferred) {
|
|
1321
|
+
const match = versions.find((v) => v.num === preferred && !v.yanked);
|
|
1322
|
+
if (match?.num) return {
|
|
1323
|
+
version: preferred,
|
|
1324
|
+
entry: match
|
|
1325
|
+
};
|
|
1326
|
+
if (versions.length === 0) return { version: preferred };
|
|
1327
|
+
}
|
|
1328
|
+
const firstStable = versions.find((v) => !v.yanked && v.num);
|
|
1329
|
+
if (firstStable?.num) return {
|
|
1330
|
+
version: firstStable.num,
|
|
1331
|
+
entry: firstStable
|
|
1332
|
+
};
|
|
1489
1333
|
return null;
|
|
1490
1334
|
}
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1335
|
+
function pickPreferredUrl(...urls) {
|
|
1336
|
+
return urls.map((v) => v?.trim()).find((v) => !!v);
|
|
1337
|
+
}
|
|
1338
|
+
async function fetchCratesApi(url) {
|
|
1339
|
+
return runCratesApiRateLimited(() => $fetch(url).catch(() => null));
|
|
1340
|
+
}
|
|
1341
|
+
async function resolveCrateDocsWithAttempts(crateName, options = {}) {
|
|
1342
|
+
const attempts = [];
|
|
1343
|
+
const onProgress = options.onProgress;
|
|
1344
|
+
const normalizedName = crateName.trim().toLowerCase();
|
|
1345
|
+
if (!normalizedName || !VALID_CRATE_NAME.test(normalizedName)) {
|
|
1346
|
+
attempts.push({
|
|
1347
|
+
source: "crates",
|
|
1348
|
+
status: "error",
|
|
1349
|
+
message: `Invalid crate name: ${crateName}`
|
|
1350
|
+
});
|
|
1351
|
+
return {
|
|
1352
|
+
package: null,
|
|
1353
|
+
attempts
|
|
1354
|
+
};
|
|
1495
1355
|
}
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1356
|
+
onProgress?.("crates.io metadata");
|
|
1357
|
+
const apiUrl = `https://crates.io/api/v1/crates/${encodeURIComponent(normalizedName)}`;
|
|
1358
|
+
const data = await fetchCratesApi(apiUrl);
|
|
1359
|
+
if (!data?.crate) {
|
|
1360
|
+
attempts.push({
|
|
1361
|
+
source: "crates",
|
|
1362
|
+
url: apiUrl,
|
|
1363
|
+
status: "not-found",
|
|
1364
|
+
message: "Crate not found on crates.io"
|
|
1365
|
+
});
|
|
1366
|
+
return {
|
|
1367
|
+
package: null,
|
|
1368
|
+
attempts
|
|
1369
|
+
};
|
|
1503
1370
|
}
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1371
|
+
attempts.push({
|
|
1372
|
+
source: "crates",
|
|
1373
|
+
url: apiUrl,
|
|
1374
|
+
status: "success",
|
|
1375
|
+
message: `Found crate: ${data.crate.name || normalizedName}`
|
|
1376
|
+
});
|
|
1377
|
+
const selected = selectCrateVersion(data, options.version);
|
|
1378
|
+
if (!selected) {
|
|
1379
|
+
attempts.push({
|
|
1380
|
+
source: "crates",
|
|
1381
|
+
url: apiUrl,
|
|
1382
|
+
status: "error",
|
|
1383
|
+
message: "No usable crate versions found"
|
|
1384
|
+
});
|
|
1385
|
+
return {
|
|
1386
|
+
package: null,
|
|
1387
|
+
attempts
|
|
1388
|
+
};
|
|
1389
|
+
}
|
|
1390
|
+
const version = selected.version;
|
|
1391
|
+
const versionEntry = selected.entry;
|
|
1392
|
+
const docsRsUrl = `https://docs.rs/${encodeURIComponent(normalizedName)}/${encodeURIComponent(version)}`;
|
|
1393
|
+
const repositoryRaw = pickPreferredUrl(versionEntry?.repository, data.crate.repository);
|
|
1394
|
+
const homepage = pickPreferredUrl(versionEntry?.homepage, data.crate.homepage);
|
|
1395
|
+
const documentation = pickPreferredUrl(versionEntry?.documentation, data.crate.documentation);
|
|
1396
|
+
const normalizedRepo = repositoryRaw ? normalizeRepoUrl(repositoryRaw) : void 0;
|
|
1397
|
+
const repoUrl = normalizedRepo && isLikelyCodeHostUrl(normalizedRepo) ? normalizedRepo : isLikelyCodeHostUrl(homepage) ? homepage : void 0;
|
|
1398
|
+
let resolved = {
|
|
1399
|
+
name: normalizedName,
|
|
1400
|
+
version,
|
|
1401
|
+
releasedAt: versionEntry?.created_at || data.crate.updated_at || void 0,
|
|
1402
|
+
description: versionEntry?.description || data.crate.description,
|
|
1403
|
+
docsUrl: (() => {
|
|
1404
|
+
if (documentation && !isUselessDocsUrl(documentation) && !isLikelyCodeHostUrl(documentation)) return documentation;
|
|
1405
|
+
if (homepage && !isUselessDocsUrl(homepage) && !isLikelyCodeHostUrl(homepage)) return homepage;
|
|
1406
|
+
return docsRsUrl;
|
|
1407
|
+
})(),
|
|
1408
|
+
repoUrl
|
|
1409
|
+
};
|
|
1410
|
+
const gh = repoUrl ? parseGitHubUrl(repoUrl) : null;
|
|
1411
|
+
if (gh) {
|
|
1412
|
+
onProgress?.("GitHub enrichment");
|
|
1413
|
+
const ghResolved = await resolveGitHubRepo(gh.owner, gh.repo);
|
|
1414
|
+
if (ghResolved) {
|
|
1415
|
+
attempts.push({
|
|
1416
|
+
source: "github-meta",
|
|
1417
|
+
url: repoUrl,
|
|
1418
|
+
status: "success",
|
|
1419
|
+
message: "Enriched via GitHub repo metadata"
|
|
1420
|
+
});
|
|
1421
|
+
resolved = {
|
|
1422
|
+
...ghResolved,
|
|
1423
|
+
name: normalizedName,
|
|
1424
|
+
version,
|
|
1425
|
+
releasedAt: resolved.releasedAt || ghResolved.releasedAt,
|
|
1426
|
+
description: resolved.description || ghResolved.description,
|
|
1427
|
+
docsUrl: resolved.docsUrl || ghResolved.docsUrl,
|
|
1428
|
+
repoUrl,
|
|
1429
|
+
readmeUrl: ghResolved.readmeUrl || resolved.readmeUrl
|
|
1430
|
+
};
|
|
1431
|
+
} else attempts.push({
|
|
1432
|
+
source: "github-meta",
|
|
1433
|
+
url: repoUrl,
|
|
1434
|
+
status: "not-found",
|
|
1435
|
+
message: "GitHub enrichment failed, using crates.io metadata"
|
|
1436
|
+
});
|
|
1437
|
+
}
|
|
1438
|
+
if (!resolved.llmsUrl && resolved.docsUrl) {
|
|
1439
|
+
onProgress?.("llms.txt discovery");
|
|
1440
|
+
resolved.llmsUrl = await fetchLlmsUrl(resolved.docsUrl).catch(() => null) ?? void 0;
|
|
1441
|
+
if (resolved.llmsUrl) attempts.push({
|
|
1442
|
+
source: "llms.txt",
|
|
1443
|
+
url: resolved.llmsUrl,
|
|
1444
|
+
status: "success"
|
|
1445
|
+
});
|
|
1446
|
+
}
|
|
1447
|
+
return {
|
|
1448
|
+
package: resolved,
|
|
1449
|
+
attempts
|
|
1450
|
+
};
|
|
1509
1451
|
}
|
|
1510
|
-
function
|
|
1511
|
-
|
|
1452
|
+
async function fetchCrawledDocs(url, onProgress, maxPages = 200) {
|
|
1453
|
+
const outputDir = join(tmpdir(), "skilld-crawl", Date.now().toString());
|
|
1454
|
+
onProgress?.(`Crawling ${url}`);
|
|
1455
|
+
const userLang = getUserLang();
|
|
1456
|
+
const foreignUrls = /* @__PURE__ */ new Set();
|
|
1457
|
+
const doCrawl = () => crawlAndGenerate({
|
|
1458
|
+
urls: [url],
|
|
1459
|
+
outputDir,
|
|
1460
|
+
driver: "http",
|
|
1461
|
+
generateLlmsTxt: false,
|
|
1462
|
+
generateIndividualMd: true,
|
|
1463
|
+
maxRequestsPerCrawl: maxPages,
|
|
1464
|
+
onPage: (page) => {
|
|
1465
|
+
const lang = extractHtmlLang(page.html);
|
|
1466
|
+
if (lang && !lang.startsWith("en") && !lang.startsWith(userLang)) foreignUrls.add(page.url);
|
|
1467
|
+
}
|
|
1468
|
+
}, (progress) => {
|
|
1469
|
+
if (progress.crawling.status === "processing" && progress.crawling.total > 0) onProgress?.(`Crawling ${progress.crawling.processed}/${progress.crawling.total} pages`);
|
|
1470
|
+
});
|
|
1471
|
+
let results = await doCrawl().catch((err) => {
|
|
1472
|
+
onProgress?.(`Crawl failed: ${err?.message || err}`);
|
|
1473
|
+
return [];
|
|
1474
|
+
});
|
|
1475
|
+
if (results.length === 0) {
|
|
1476
|
+
onProgress?.("Retrying crawl");
|
|
1477
|
+
results = await doCrawl().catch(() => []);
|
|
1478
|
+
}
|
|
1479
|
+
rmSync(outputDir, {
|
|
1480
|
+
recursive: true,
|
|
1481
|
+
force: true
|
|
1482
|
+
});
|
|
1483
|
+
const docs = [];
|
|
1484
|
+
let localeFiltered = 0;
|
|
1485
|
+
for (const result of results) {
|
|
1486
|
+
if (!result.success || !result.content) continue;
|
|
1487
|
+
if (foreignUrls.has(result.url)) {
|
|
1488
|
+
localeFiltered++;
|
|
1489
|
+
continue;
|
|
1490
|
+
}
|
|
1491
|
+
const segments = (new URL(result.url).pathname.replace(/\/$/, "") || "/index").split("/").filter(Boolean);
|
|
1492
|
+
if (isForeignPathPrefix(segments[0], userLang)) {
|
|
1493
|
+
localeFiltered++;
|
|
1494
|
+
continue;
|
|
1495
|
+
}
|
|
1496
|
+
const path = `docs/${segments.join("/")}.md`;
|
|
1497
|
+
docs.push({
|
|
1498
|
+
path,
|
|
1499
|
+
content: result.content
|
|
1500
|
+
});
|
|
1501
|
+
}
|
|
1502
|
+
if (localeFiltered > 0) onProgress?.(`Filtered ${localeFiltered} foreign locale pages`);
|
|
1503
|
+
onProgress?.(`Crawled ${docs.length} pages`);
|
|
1504
|
+
return docs;
|
|
1512
1505
|
}
|
|
1513
|
-
const
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
"solid",
|
|
1517
|
-
"angular",
|
|
1518
|
-
"svelte",
|
|
1519
|
-
"preact",
|
|
1520
|
-
"lit",
|
|
1521
|
-
"qwik"
|
|
1522
|
-
]);
|
|
1523
|
-
function filterFrameworkDocs(files, packageName) {
|
|
1524
|
-
if (!packageName) return files;
|
|
1525
|
-
const shortName = packageName.replace(/^@.*\//, "");
|
|
1526
|
-
const targetFramework = [...FRAMEWORK_NAMES].find((fw) => shortName.includes(fw));
|
|
1527
|
-
if (!targetFramework) return files;
|
|
1528
|
-
const otherFrameworks = [...FRAMEWORK_NAMES].filter((fw) => fw !== targetFramework);
|
|
1529
|
-
const excludePattern = new RegExp(`\\b(?:${otherFrameworks.join("|")})\\b`);
|
|
1530
|
-
return files.filter((f) => !excludePattern.test(f));
|
|
1506
|
+
const HTML_LANG_RE = /<html[^>]*\slang=["']([^"']+)["']/i;
|
|
1507
|
+
function extractHtmlLang(html) {
|
|
1508
|
+
return HTML_LANG_RE.exec(html)?.[1]?.toLowerCase();
|
|
1531
1509
|
}
|
|
1532
|
-
const
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
"
|
|
1540
|
-
"
|
|
1541
|
-
"
|
|
1542
|
-
"
|
|
1543
|
-
"
|
|
1544
|
-
"
|
|
1545
|
-
"
|
|
1546
|
-
"
|
|
1547
|
-
"
|
|
1548
|
-
"
|
|
1549
|
-
"
|
|
1550
|
-
"
|
|
1551
|
-
"
|
|
1552
|
-
"
|
|
1553
|
-
"mocks",
|
|
1554
|
-
"__mocks__"
|
|
1555
|
-
]);
|
|
1556
|
-
const DOC_DIR_BONUS = new Set([
|
|
1557
|
-
"docs",
|
|
1558
|
-
"documentation",
|
|
1559
|
-
"pages",
|
|
1560
|
-
"content",
|
|
1561
|
-
"website",
|
|
1562
|
-
"guide",
|
|
1563
|
-
"guides",
|
|
1564
|
-
"wiki",
|
|
1565
|
-
"manual",
|
|
1566
|
-
"api"
|
|
1510
|
+
const LOCALE_CODES = new Set([
|
|
1511
|
+
"ar",
|
|
1512
|
+
"de",
|
|
1513
|
+
"es",
|
|
1514
|
+
"fr",
|
|
1515
|
+
"id",
|
|
1516
|
+
"it",
|
|
1517
|
+
"ja",
|
|
1518
|
+
"ko",
|
|
1519
|
+
"nl",
|
|
1520
|
+
"pl",
|
|
1521
|
+
"pt",
|
|
1522
|
+
"pt-br",
|
|
1523
|
+
"ru",
|
|
1524
|
+
"th",
|
|
1525
|
+
"tr",
|
|
1526
|
+
"uk",
|
|
1527
|
+
"vi",
|
|
1528
|
+
"zh",
|
|
1529
|
+
"zh-cn",
|
|
1530
|
+
"zh-tw"
|
|
1567
1531
|
]);
|
|
1568
|
-
function
|
|
1569
|
-
|
|
1532
|
+
function isForeignPathPrefix(segment, userLang) {
|
|
1533
|
+
if (!segment) return false;
|
|
1534
|
+
const lower = segment.toLowerCase();
|
|
1535
|
+
if (lower === "en" || lower.startsWith(userLang)) return false;
|
|
1536
|
+
return LOCALE_CODES.has(lower);
|
|
1570
1537
|
}
|
|
1571
|
-
function
|
|
1572
|
-
|
|
1538
|
+
function getUserLang() {
|
|
1539
|
+
const code = (process.env.LC_ALL || process.env.LANG || process.env.LANGUAGE || "").split(/[_.:-]/)[0]?.toLowerCase() || "";
|
|
1540
|
+
return code.length >= 2 ? code.slice(0, 2) : "en";
|
|
1573
1541
|
}
|
|
1574
|
-
function
|
|
1575
|
-
return
|
|
1542
|
+
function toCrawlPattern(docsUrl) {
|
|
1543
|
+
return `${docsUrl.replace(/\/+$/, "")}/**`;
|
|
1576
1544
|
}
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1545
|
+
const HIGH_VALUE_CATEGORIES = new Set([
|
|
1546
|
+
"q&a",
|
|
1547
|
+
"help",
|
|
1548
|
+
"troubleshooting",
|
|
1549
|
+
"support"
|
|
1550
|
+
]);
|
|
1551
|
+
const LOW_VALUE_CATEGORIES = new Set([
|
|
1552
|
+
"show and tell",
|
|
1553
|
+
"ideas",
|
|
1554
|
+
"polls"
|
|
1555
|
+
]);
|
|
1556
|
+
const TITLE_NOISE_RE = /looking .*(?:developer|engineer|freelanc)|hiring|job post|guide me to (?:complete|finish|build)|help me (?:complete|finish|build)|seeking .* tutorial|recommend.* course/i;
|
|
1557
|
+
const MIN_DISCUSSION_SCORE = 3;
|
|
1558
|
+
function scoreComment(c) {
|
|
1559
|
+
return (c.isMaintainer ? 3 : 1) * (hasCodeBlock(c.body) ? 2 : 1) * (1 + c.reactions);
|
|
1580
1560
|
}
|
|
1581
|
-
function
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1561
|
+
function scoreDiscussion(d) {
|
|
1562
|
+
if (TITLE_NOISE_RE.test(d.title)) return -1;
|
|
1563
|
+
let score = 0;
|
|
1564
|
+
if (d.isMaintainer) score += 3;
|
|
1565
|
+
if (hasCodeBlock([
|
|
1566
|
+
d.body,
|
|
1567
|
+
d.answer || "",
|
|
1568
|
+
...d.topComments.map((c) => c.body)
|
|
1569
|
+
].join("\n"))) score += 3;
|
|
1570
|
+
score += Math.min(d.upvoteCount, 5);
|
|
1571
|
+
if (d.answer) {
|
|
1572
|
+
score += 2;
|
|
1573
|
+
if (d.answer.length > 100) score += 1;
|
|
1590
1574
|
}
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1575
|
+
if (d.topComments.some((c) => c.isMaintainer)) score += 2;
|
|
1576
|
+
if (d.topComments.some((c) => c.reactions > 0)) score += 1;
|
|
1577
|
+
return score;
|
|
1578
|
+
}
|
|
1579
|
+
async function fetchGitHubDiscussions(owner, repo, limit = 20, releasedAt, fromDate) {
|
|
1580
|
+
if (!isGhAvailable()) return [];
|
|
1581
|
+
if (!fromDate && releasedAt) {
|
|
1582
|
+
const cutoff = new Date(releasedAt);
|
|
1583
|
+
cutoff.setMonth(cutoff.getMonth() + 6);
|
|
1584
|
+
if (cutoff < /* @__PURE__ */ new Date()) return [];
|
|
1596
1585
|
}
|
|
1597
|
-
|
|
1598
|
-
const
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1586
|
+
try {
|
|
1587
|
+
const { stdout: result } = spawnSync("gh", [
|
|
1588
|
+
"api",
|
|
1589
|
+
"graphql",
|
|
1590
|
+
"-f",
|
|
1591
|
+
`query=${`query($owner: String!, $repo: String!) { repository(owner: $owner, name: $repo) { discussions(first: ${Math.min(limit * 3, 80)}, orderBy: {field: CREATED_AT, direction: DESC}) { nodes { number title body category { name } createdAt url upvoteCount comments(first: 10) { totalCount nodes { body author { login } authorAssociation reactions { totalCount } } } answer { body author { login } authorAssociation } author { login } authorAssociation } } } }`}`,
|
|
1592
|
+
"-f",
|
|
1593
|
+
`owner=${owner}`,
|
|
1594
|
+
"-f",
|
|
1595
|
+
`repo=${repo}`
|
|
1596
|
+
], {
|
|
1597
|
+
encoding: "utf-8",
|
|
1598
|
+
maxBuffer: 10 * 1024 * 1024
|
|
1599
|
+
});
|
|
1600
|
+
if (!result) return [];
|
|
1601
|
+
const nodes = JSON.parse(result)?.data?.repository?.discussions?.nodes;
|
|
1602
|
+
if (!Array.isArray(nodes)) return [];
|
|
1603
|
+
const fromTs = fromDate ? new Date(fromDate).getTime() : null;
|
|
1604
|
+
return nodes.filter((d) => d.author && !BOT_USERS.has(d.author.login)).filter((d) => {
|
|
1605
|
+
const cat = (d.category?.name || "").toLowerCase();
|
|
1606
|
+
return !LOW_VALUE_CATEGORIES.has(cat);
|
|
1607
|
+
}).filter((d) => !fromTs || new Date(d.createdAt).getTime() >= fromTs).map((d) => {
|
|
1608
|
+
let answer;
|
|
1609
|
+
if (d.answer?.body) {
|
|
1610
|
+
const isMaintainer = [
|
|
1611
|
+
"OWNER",
|
|
1612
|
+
"MEMBER",
|
|
1613
|
+
"COLLABORATOR"
|
|
1614
|
+
].includes(d.answer.authorAssociation);
|
|
1615
|
+
const author = d.answer.author?.login;
|
|
1616
|
+
answer = `${isMaintainer && author ? `**@${author}** [maintainer]:\n\n` : ""}${d.answer.body}`;
|
|
1617
|
+
}
|
|
1618
|
+
const comments = (d.comments?.nodes || []).filter((c) => c.author && !BOT_USERS.has(c.author.login)).filter((c) => !COMMENT_NOISE_RE.test((c.body || "").trim())).map((c) => {
|
|
1619
|
+
const isMaintainer = [
|
|
1620
|
+
"OWNER",
|
|
1621
|
+
"MEMBER",
|
|
1622
|
+
"COLLABORATOR"
|
|
1623
|
+
].includes(c.authorAssociation);
|
|
1624
|
+
return {
|
|
1625
|
+
body: c.body || "",
|
|
1626
|
+
author: c.author.login,
|
|
1627
|
+
reactions: c.reactions?.totalCount || 0,
|
|
1628
|
+
isMaintainer
|
|
1629
|
+
};
|
|
1630
|
+
}).sort((a, b) => scoreComment(b) - scoreComment(a)).slice(0, 3);
|
|
1603
1631
|
return {
|
|
1604
|
-
|
|
1605
|
-
|
|
1632
|
+
number: d.number,
|
|
1633
|
+
title: d.title,
|
|
1634
|
+
body: d.body || "",
|
|
1635
|
+
category: d.category?.name || "",
|
|
1636
|
+
createdAt: d.createdAt,
|
|
1637
|
+
url: d.url,
|
|
1638
|
+
upvoteCount: d.upvoteCount || 0,
|
|
1639
|
+
comments: d.comments?.totalCount || 0,
|
|
1640
|
+
isMaintainer: [
|
|
1641
|
+
"OWNER",
|
|
1642
|
+
"MEMBER",
|
|
1643
|
+
"COLLABORATOR"
|
|
1644
|
+
].includes(d.authorAssociation),
|
|
1645
|
+
answer,
|
|
1646
|
+
topComments: comments
|
|
1606
1647
|
};
|
|
1607
|
-
}
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1648
|
+
}).map((d) => ({
|
|
1649
|
+
d,
|
|
1650
|
+
score: scoreDiscussion(d)
|
|
1651
|
+
})).filter(({ score }) => score >= MIN_DISCUSSION_SCORE).sort((a, b) => {
|
|
1652
|
+
const aHigh = HIGH_VALUE_CATEGORIES.has(a.d.category.toLowerCase()) ? 1 : 0;
|
|
1653
|
+
const bHigh = HIGH_VALUE_CATEGORIES.has(b.d.category.toLowerCase()) ? 1 : 0;
|
|
1654
|
+
if (aHigh !== bHigh) return bHigh - aHigh;
|
|
1655
|
+
return b.score - a.score;
|
|
1656
|
+
}).slice(0, limit).map(({ d }) => d);
|
|
1657
|
+
} catch {
|
|
1658
|
+
return [];
|
|
1615
1659
|
}
|
|
1616
|
-
if (dirGroups.size === 0) return null;
|
|
1617
|
-
const scored = Array.from(dirGroups.entries(), ([dir, files]) => ({
|
|
1618
|
-
dir,
|
|
1619
|
-
files,
|
|
1620
|
-
score: scoreDocDir(dir, files.length)
|
|
1621
|
-
})).filter((d) => d.files.length >= 5).sort((a, b) => b.score - a.score);
|
|
1622
|
-
if (scored.length === 0) return null;
|
|
1623
|
-
const best = scored[0];
|
|
1624
|
-
return {
|
|
1625
|
-
files: best.files,
|
|
1626
|
-
prefix: best.dir
|
|
1627
|
-
};
|
|
1628
|
-
}
|
|
1629
|
-
async function listDocsAtRef(owner, repo, ref, pathPrefix = "docs/") {
|
|
1630
|
-
return filterDocFiles(await listFilesAtRef(owner, repo, ref), pathPrefix);
|
|
1631
1660
|
}
|
|
1632
|
-
|
|
1633
|
-
const
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
1649
|
-
|
|
1650
|
-
|
|
1651
|
-
|
|
1652
|
-
|
|
1653
|
-
const
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
allFiles = tag.files;
|
|
1661
|
+
function formatDiscussionAsMarkdown(d) {
|
|
1662
|
+
const fm = buildFrontmatter({
|
|
1663
|
+
number: d.number,
|
|
1664
|
+
title: d.title,
|
|
1665
|
+
category: d.category,
|
|
1666
|
+
created: isoDate(d.createdAt),
|
|
1667
|
+
url: d.url,
|
|
1668
|
+
upvotes: d.upvoteCount,
|
|
1669
|
+
comments: d.comments,
|
|
1670
|
+
answered: !!d.answer
|
|
1671
|
+
});
|
|
1672
|
+
const bodyLimit = d.upvoteCount >= 5 ? 1500 : 800;
|
|
1673
|
+
const lines = [
|
|
1674
|
+
fm,
|
|
1675
|
+
"",
|
|
1676
|
+
`# ${d.title}`
|
|
1677
|
+
];
|
|
1678
|
+
if (d.body) lines.push("", truncateBody(d.body, bodyLimit));
|
|
1679
|
+
if (d.answer) lines.push("", "---", "", "## Accepted Answer", "", truncateBody(d.answer, 1e3));
|
|
1680
|
+
else if (d.topComments.length > 0) {
|
|
1681
|
+
lines.push("", "---", "", "## Top Comments");
|
|
1682
|
+
for (const c of d.topComments) {
|
|
1683
|
+
const reactions = c.reactions > 0 ? ` (+${c.reactions})` : "";
|
|
1684
|
+
const maintainer = c.isMaintainer ? " [maintainer]" : "";
|
|
1685
|
+
lines.push("", `**@${c.author}**${maintainer}${reactions}:`, "", truncateBody(c.body, 600));
|
|
1658
1686
|
}
|
|
1659
1687
|
}
|
|
1660
|
-
|
|
1661
|
-
if (docs.length === 0) return null;
|
|
1662
|
-
return {
|
|
1663
|
-
baseUrl: `https://raw.githubusercontent.com/${owner}/${repo}/${tag.ref}`,
|
|
1664
|
-
ref: tag.ref,
|
|
1665
|
-
files: docs,
|
|
1666
|
-
docsPrefix,
|
|
1667
|
-
allFiles,
|
|
1668
|
-
fallback: tag.fallback
|
|
1669
|
-
};
|
|
1670
|
-
}
|
|
1671
|
-
function normalizePath(p) {
|
|
1672
|
-
return p.replace(/^\//, "").replace(/\.(?:md|mdx)$/, "");
|
|
1688
|
+
return lines.join("\n");
|
|
1673
1689
|
}
|
|
1674
|
-
function
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1690
|
+
function generateDiscussionIndex(discussions) {
|
|
1691
|
+
const byCategory = /* @__PURE__ */ new Map();
|
|
1692
|
+
for (const d of discussions) mapInsert(byCategory, d.category || "Uncategorized", () => []).push(d);
|
|
1693
|
+
const answered = discussions.filter((d) => d.answer).length;
|
|
1694
|
+
const sections = [
|
|
1695
|
+
[
|
|
1696
|
+
"---",
|
|
1697
|
+
`total: ${discussions.length}`,
|
|
1698
|
+
`answered: ${answered}`,
|
|
1699
|
+
"---"
|
|
1700
|
+
].join("\n"),
|
|
1701
|
+
"",
|
|
1702
|
+
"# Discussions Index",
|
|
1703
|
+
""
|
|
1704
|
+
];
|
|
1705
|
+
const cats = [...byCategory.keys()].sort((a, b) => {
|
|
1706
|
+
return (HIGH_VALUE_CATEGORIES.has(a.toLowerCase()) ? 0 : 1) - (HIGH_VALUE_CATEGORIES.has(b.toLowerCase()) ? 0 : 1) || a.localeCompare(b);
|
|
1686
1707
|
});
|
|
1687
|
-
const
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1708
|
+
for (const cat of cats) {
|
|
1709
|
+
const group = byCategory.get(cat);
|
|
1710
|
+
sections.push(`## ${cat} (${group.length})`, "");
|
|
1711
|
+
for (const d of group) {
|
|
1712
|
+
const upvotes = d.upvoteCount > 0 ? ` (+${d.upvoteCount})` : "";
|
|
1713
|
+
const answered = d.answer ? " [answered]" : "";
|
|
1714
|
+
const date = isoDate(d.createdAt);
|
|
1715
|
+
sections.push(`- [#${d.number}](./discussion-${d.number}.md): ${d.title}${upvotes}${answered} (${date})`);
|
|
1716
|
+
}
|
|
1717
|
+
sections.push("");
|
|
1692
1718
|
}
|
|
1693
|
-
|
|
1694
|
-
return {
|
|
1695
|
-
isValid: matchRatio >= .3,
|
|
1696
|
-
matchRatio
|
|
1697
|
-
};
|
|
1719
|
+
return sections.join("\n");
|
|
1698
1720
|
}
|
|
1699
|
-
|
|
1700
|
-
const
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
|
|
1721
|
+
function generateDocsIndex(docs) {
|
|
1722
|
+
const docFiles = docs.filter((d) => d.path.startsWith("docs/") && d.path.endsWith(".md") && !d.path.endsWith("_INDEX.md")).sort((a, b) => a.path.localeCompare(b.path));
|
|
1723
|
+
if (docFiles.length === 0) return "";
|
|
1724
|
+
const rootFiles = [];
|
|
1725
|
+
const byDir = /* @__PURE__ */ new Map();
|
|
1726
|
+
for (const doc of docFiles) {
|
|
1727
|
+
const rel = doc.path.slice(5);
|
|
1728
|
+
const dir = rel.includes("/") ? rel.slice(0, rel.lastIndexOf("/")) : "";
|
|
1729
|
+
if (!dir) rootFiles.push(doc);
|
|
1730
|
+
else {
|
|
1731
|
+
const list = byDir.get(dir);
|
|
1732
|
+
if (list) list.push(doc);
|
|
1733
|
+
else byDir.set(dir, [doc]);
|
|
1734
|
+
}
|
|
1735
|
+
}
|
|
1736
|
+
const sections = [
|
|
1737
|
+
"---",
|
|
1738
|
+
`total: ${docFiles.length}`,
|
|
1739
|
+
"---",
|
|
1740
|
+
"",
|
|
1741
|
+
"# Docs Index",
|
|
1742
|
+
""
|
|
1705
1743
|
];
|
|
1706
|
-
for (const
|
|
1707
|
-
const
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1744
|
+
for (const file of rootFiles) {
|
|
1745
|
+
const rel = file.path.slice(5);
|
|
1746
|
+
const title = extractTitle(file.content) || rel.replace(/\.md$/, "");
|
|
1747
|
+
const desc = extractDescription(file.content);
|
|
1748
|
+
const descPart = desc ? `: ${desc}` : "";
|
|
1749
|
+
sections.push(`- [${title}](./${rel})${descPart}`);
|
|
1712
1750
|
}
|
|
1713
|
-
|
|
1751
|
+
if (rootFiles.length > 0) sections.push("");
|
|
1752
|
+
for (const [dir, files] of byDir) {
|
|
1753
|
+
sections.push(`## ${dir} (${files.length})`, "");
|
|
1754
|
+
for (const file of files) {
|
|
1755
|
+
const rel = file.path.slice(5);
|
|
1756
|
+
const title = extractTitle(file.content) || rel.replace(/\.md$/, "").split("/").pop();
|
|
1757
|
+
const desc = extractDescription(file.content);
|
|
1758
|
+
const descPart = desc ? `: ${desc}` : "";
|
|
1759
|
+
sections.push(`- [${title}](./${rel})${descPart}`);
|
|
1760
|
+
}
|
|
1761
|
+
sections.push("");
|
|
1762
|
+
}
|
|
1763
|
+
return sections.join("\n");
|
|
1714
1764
|
}
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1765
|
+
const SKIP_DIRS = [
|
|
1766
|
+
"node_modules",
|
|
1767
|
+
"_vendor",
|
|
1768
|
+
"__tests__",
|
|
1769
|
+
"__mocks__",
|
|
1770
|
+
"__fixtures__",
|
|
1771
|
+
"test",
|
|
1772
|
+
"tests",
|
|
1773
|
+
"fixture",
|
|
1774
|
+
"fixtures",
|
|
1775
|
+
"locales",
|
|
1776
|
+
"locale",
|
|
1777
|
+
"i18n",
|
|
1778
|
+
".git"
|
|
1779
|
+
];
|
|
1780
|
+
const SKIP_PATTERNS = [
|
|
1781
|
+
"*.min.*",
|
|
1782
|
+
"*.prod.*",
|
|
1783
|
+
"*.global.*",
|
|
1784
|
+
"*.browser.*",
|
|
1785
|
+
"*.map",
|
|
1786
|
+
"*.map.js",
|
|
1787
|
+
"CHANGELOG*",
|
|
1788
|
+
"LICENSE*",
|
|
1789
|
+
"README*"
|
|
1790
|
+
];
|
|
1791
|
+
const MAX_FILE_SIZE = 500 * 1024;
|
|
1792
|
+
async function resolveEntryFiles(packageDir) {
|
|
1793
|
+
if (!existsSync(join(packageDir, "package.json"))) return [];
|
|
1794
|
+
const files = await glob(["**/*.d.{ts,mts,cts}"], {
|
|
1795
|
+
cwd: packageDir,
|
|
1796
|
+
ignore: [...SKIP_DIRS.map((d) => `**/${d}/**`), ...SKIP_PATTERNS],
|
|
1797
|
+
absolute: false,
|
|
1798
|
+
expandDirectories: false
|
|
1799
|
+
});
|
|
1800
|
+
const entries = [];
|
|
1801
|
+
for (const file of files) {
|
|
1802
|
+
const absPath = join(packageDir, file);
|
|
1803
|
+
let content;
|
|
1804
|
+
try {
|
|
1805
|
+
content = readFileSync(absPath, "utf-8");
|
|
1806
|
+
} catch {
|
|
1720
1807
|
continue;
|
|
1721
1808
|
}
|
|
1722
|
-
if (
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
"search",
|
|
1728
|
-
"repos",
|
|
1729
|
-
searchTerm,
|
|
1730
|
-
"--json",
|
|
1731
|
-
"fullName",
|
|
1732
|
-
"--limit",
|
|
1733
|
-
"5"
|
|
1734
|
-
], {
|
|
1735
|
-
encoding: "utf-8",
|
|
1736
|
-
timeout: 15e3
|
|
1809
|
+
if (content.length > MAX_FILE_SIZE) continue;
|
|
1810
|
+
entries.push({
|
|
1811
|
+
path: file,
|
|
1812
|
+
content,
|
|
1813
|
+
type: "types"
|
|
1737
1814
|
});
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
|
|
1743
|
-
|
|
1744
|
-
|
|
1815
|
+
}
|
|
1816
|
+
return entries;
|
|
1817
|
+
}
|
|
1818
|
+
function parseGitSkillInput(input) {
|
|
1819
|
+
const trimmed = input.trim();
|
|
1820
|
+
if (trimmed.startsWith("@")) return null;
|
|
1821
|
+
if (trimmed.startsWith("./") || trimmed.startsWith("../") || trimmed.startsWith("/") || trimmed.startsWith("~")) return {
|
|
1822
|
+
type: "local",
|
|
1823
|
+
localPath: trimmed.startsWith("~") ? resolve(process.env.HOME || "", trimmed.slice(1)) : resolve(trimmed)
|
|
1824
|
+
};
|
|
1825
|
+
if (trimmed.startsWith("git@")) {
|
|
1826
|
+
const gh = parseGitHubUrl(normalizeRepoUrl(trimmed));
|
|
1827
|
+
if (gh) return {
|
|
1828
|
+
type: "github",
|
|
1829
|
+
owner: gh.owner,
|
|
1830
|
+
repo: gh.repo
|
|
1831
|
+
};
|
|
1832
|
+
return null;
|
|
1833
|
+
}
|
|
1834
|
+
if (trimmed.startsWith("https://") || trimmed.startsWith("http://")) return parseGitUrl(trimmed);
|
|
1835
|
+
if (/^[\w.-]+\/[\w.-]+$/.test(trimmed)) return {
|
|
1836
|
+
type: "github",
|
|
1837
|
+
owner: trimmed.split("/")[0],
|
|
1838
|
+
repo: trimmed.split("/")[1]
|
|
1839
|
+
};
|
|
1840
|
+
return null;
|
|
1841
|
+
}
|
|
1842
|
+
function parseGitUrl(url) {
|
|
1843
|
+
try {
|
|
1844
|
+
const parsed = new URL(url);
|
|
1845
|
+
if (parsed.hostname === "github.com" || parsed.hostname === "www.github.com") {
|
|
1846
|
+
const parts = parsed.pathname.replace(/^\//, "").replace(/\.git$/, "").split("/");
|
|
1847
|
+
const owner = parts[0];
|
|
1848
|
+
const repo = parts[1];
|
|
1849
|
+
if (!owner || !repo) return null;
|
|
1850
|
+
if (parts[2] === "tree" && parts.length >= 4) return {
|
|
1851
|
+
type: "github",
|
|
1852
|
+
owner,
|
|
1853
|
+
repo,
|
|
1854
|
+
ref: parts[3],
|
|
1855
|
+
skillPath: parts.length > 4 ? parts.slice(4).join("/") : void 0
|
|
1856
|
+
};
|
|
1857
|
+
return {
|
|
1858
|
+
type: "github",
|
|
1859
|
+
owner,
|
|
1860
|
+
repo
|
|
1861
|
+
};
|
|
1745
1862
|
}
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1863
|
+
if (parsed.hostname === "gitlab.com") {
|
|
1864
|
+
const parts = parsed.pathname.replace(/^\//, "").replace(/\.git$/, "").split("/");
|
|
1865
|
+
const owner = parts[0];
|
|
1866
|
+
const repo = parts[1];
|
|
1867
|
+
if (!owner || !repo) return null;
|
|
1868
|
+
return {
|
|
1869
|
+
type: "gitlab",
|
|
1870
|
+
owner,
|
|
1871
|
+
repo
|
|
1872
|
+
};
|
|
1873
|
+
}
|
|
1874
|
+
return null;
|
|
1875
|
+
} catch {
|
|
1876
|
+
return null;
|
|
1754
1877
|
}
|
|
1755
|
-
return null;
|
|
1756
1878
|
}
|
|
1757
|
-
|
|
1758
|
-
const
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1879
|
+
function parseSkillFrontmatterName(content) {
|
|
1880
|
+
const fm = parseFrontmatter(content);
|
|
1881
|
+
return {
|
|
1882
|
+
name: fm.name,
|
|
1883
|
+
description: fm.description
|
|
1884
|
+
};
|
|
1762
1885
|
}
|
|
1763
|
-
|
|
1764
|
-
const
|
|
1765
|
-
if (!
|
|
1766
|
-
|
|
1767
|
-
|
|
1886
|
+
function collectFiles(dir, prefix = "") {
|
|
1887
|
+
const files = [];
|
|
1888
|
+
if (!existsSync(dir)) return files;
|
|
1889
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
1890
|
+
const relPath = prefix ? `${prefix}/${entry.name}` : entry.name;
|
|
1891
|
+
const fullPath = resolve(dir, entry.name);
|
|
1892
|
+
if (entry.isDirectory()) files.push(...collectFiles(fullPath, relPath));
|
|
1893
|
+
else if (entry.isFile()) files.push({
|
|
1894
|
+
path: relPath,
|
|
1895
|
+
content: readFileSync(fullPath, "utf-8")
|
|
1896
|
+
});
|
|
1768
1897
|
}
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1898
|
+
return files;
|
|
1899
|
+
}
|
|
1900
|
+
async function fetchGitSkills(source, onProgress) {
|
|
1901
|
+
if (source.type === "local") return fetchLocalSkills(source);
|
|
1902
|
+
if (source.type === "github") return fetchGitHubSkills(source, onProgress);
|
|
1903
|
+
if (source.type === "gitlab") return fetchGitLabSkills(source, onProgress);
|
|
1904
|
+
return { skills: [] };
|
|
1905
|
+
}
|
|
1906
|
+
function fetchLocalSkills(source) {
|
|
1907
|
+
const base = source.localPath;
|
|
1908
|
+
if (!existsSync(base)) return { skills: [] };
|
|
1909
|
+
const skills = [];
|
|
1910
|
+
const skillsDir = resolve(base, "skills");
|
|
1911
|
+
if (existsSync(skillsDir)) for (const entry of readdirSync(skillsDir, { withFileTypes: true })) {
|
|
1912
|
+
if (!entry.isDirectory()) continue;
|
|
1913
|
+
const skill = readLocalSkill(resolve(skillsDir, entry.name), `skills/${entry.name}`);
|
|
1914
|
+
if (skill) skills.push(skill);
|
|
1780
1915
|
}
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
markRepoPrivate(owner, repo);
|
|
1785
|
-
return apiData.download_url;
|
|
1916
|
+
if (skills.length === 0) {
|
|
1917
|
+
const skill = readLocalSkill(base, "");
|
|
1918
|
+
if (skill) skills.push(skill);
|
|
1786
1919
|
}
|
|
1787
|
-
return
|
|
1920
|
+
return { skills };
|
|
1788
1921
|
}
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1922
|
+
function readLocalSkill(dir, repoPath) {
|
|
1923
|
+
const skillMdPath = resolve(dir, "SKILL.md");
|
|
1924
|
+
if (!existsSync(skillMdPath)) return null;
|
|
1925
|
+
const content = readFileSync(skillMdPath, "utf-8");
|
|
1926
|
+
const frontmatter = parseSkillFrontmatterName(content);
|
|
1927
|
+
const dirName = dir.split("/").pop();
|
|
1928
|
+
const name = frontmatter.name || dirName;
|
|
1929
|
+
const files = collectFiles(dir).filter((f) => f.path !== "SKILL.md");
|
|
1930
|
+
return {
|
|
1931
|
+
name,
|
|
1932
|
+
description: frontmatter.description || "",
|
|
1933
|
+
path: repoPath,
|
|
1934
|
+
content,
|
|
1935
|
+
files
|
|
1936
|
+
};
|
|
1937
|
+
}
|
|
1938
|
+
async function fetchGitHubSkills(source, onProgress) {
|
|
1939
|
+
const { owner, repo } = source;
|
|
1940
|
+
if (!owner || !repo) return { skills: [] };
|
|
1941
|
+
const ref = source.ref || "main";
|
|
1942
|
+
const refs = ref === "main" ? ["main", "master"] : [ref];
|
|
1943
|
+
for (const tryRef of refs) {
|
|
1944
|
+
const skills = await downloadGitHubSkills(owner, repo, tryRef, source.skillPath, onProgress);
|
|
1945
|
+
if (skills.length > 0) return { skills };
|
|
1794
1946
|
}
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1947
|
+
return { skills: [] };
|
|
1948
|
+
}
|
|
1949
|
+
async function downloadGitHubSkills(owner, repo, ref, skillPath, onProgress) {
|
|
1950
|
+
const tempDir = join(tmpdir(), `skilld-${Date.now()}`);
|
|
1951
|
+
try {
|
|
1952
|
+
if (skillPath) {
|
|
1953
|
+
onProgress?.(`Downloading ${owner}/${repo}/${skillPath}@${ref}`);
|
|
1954
|
+
const { dir } = await downloadTemplate(`github:${owner}/${repo}/${skillPath}#${ref}`, {
|
|
1955
|
+
dir: tempDir,
|
|
1956
|
+
force: true,
|
|
1957
|
+
auth: getGitHubToken() || void 0
|
|
1958
|
+
});
|
|
1959
|
+
const skill = readLocalSkill(dir, skillPath);
|
|
1960
|
+
return skill ? [skill] : [];
|
|
1802
1961
|
}
|
|
1803
|
-
|
|
1804
|
-
const owner = parts[0];
|
|
1805
|
-
const repo = parts[1];
|
|
1806
|
-
const subdir = parts.slice(2).join("/");
|
|
1807
|
-
const text = await $fetch(subdir ? `https://ungh.cc/repos/${owner}/${repo}/files/${ref}/${subdir}/README.md` : `https://ungh.cc/repos/${owner}/${repo}/readme?ref=${ref}`, { responseType: "text" }).catch(() => null);
|
|
1808
|
-
if (!text) return null;
|
|
1962
|
+
onProgress?.(`Downloading ${owner}/${repo}/skills@${ref}`);
|
|
1809
1963
|
try {
|
|
1810
|
-
const
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1964
|
+
const { dir } = await downloadTemplate(`github:${owner}/${repo}/skills#${ref}`, {
|
|
1965
|
+
dir: tempDir,
|
|
1966
|
+
force: true,
|
|
1967
|
+
auth: getGitHubToken() || void 0
|
|
1968
|
+
});
|
|
1969
|
+
const skills = [];
|
|
1970
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
1971
|
+
if (!entry.isDirectory()) continue;
|
|
1972
|
+
const skill = readLocalSkill(resolve(dir, entry.name), `skills/${entry.name}`);
|
|
1973
|
+
if (skill) skills.push(skill);
|
|
1974
|
+
}
|
|
1975
|
+
if (skills.length > 0) {
|
|
1976
|
+
onProgress?.(`Found ${skills.length} skill(s)`);
|
|
1977
|
+
return skills;
|
|
1978
|
+
}
|
|
1979
|
+
} catch {}
|
|
1980
|
+
const content = await fetchGitHubRaw(`https://raw.githubusercontent.com/${owner}/${repo}/${ref}/SKILL.md`);
|
|
1981
|
+
if (content) {
|
|
1982
|
+
const fm = parseSkillFrontmatterName(content);
|
|
1983
|
+
onProgress?.("Found 1 skill");
|
|
1984
|
+
return [{
|
|
1985
|
+
name: fm.name || repo,
|
|
1986
|
+
description: fm.description || "",
|
|
1987
|
+
path: "",
|
|
1988
|
+
content,
|
|
1989
|
+
files: []
|
|
1990
|
+
}];
|
|
1814
1991
|
}
|
|
1992
|
+
return [];
|
|
1993
|
+
} catch {
|
|
1994
|
+
return [];
|
|
1995
|
+
} finally {
|
|
1996
|
+
rmSync(tempDir, {
|
|
1997
|
+
recursive: true,
|
|
1998
|
+
force: true
|
|
1999
|
+
});
|
|
1815
2000
|
}
|
|
1816
|
-
if (url.includes("raw.githubusercontent.com")) return fetchGitHubRaw(url);
|
|
1817
|
-
return fetchText(url);
|
|
1818
2001
|
}
|
|
1819
|
-
async function
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
const
|
|
1823
|
-
const
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
2002
|
+
async function fetchGitLabSkills(source, onProgress) {
|
|
2003
|
+
const { owner, repo } = source;
|
|
2004
|
+
if (!owner || !repo) return { skills: [] };
|
|
2005
|
+
const ref = source.ref || "main";
|
|
2006
|
+
const tempDir = join(tmpdir(), `skilld-gitlab-${Date.now()}`);
|
|
2007
|
+
try {
|
|
2008
|
+
const subdir = source.skillPath || "skills";
|
|
2009
|
+
onProgress?.(`Downloading ${owner}/${repo}/${subdir}@${ref}`);
|
|
2010
|
+
const { dir } = await downloadTemplate(`gitlab:${owner}/${repo}/${subdir}#${ref}`, {
|
|
2011
|
+
dir: tempDir,
|
|
2012
|
+
force: true
|
|
2013
|
+
});
|
|
2014
|
+
if (source.skillPath) {
|
|
2015
|
+
const skill = readLocalSkill(dir, source.skillPath);
|
|
2016
|
+
return { skills: skill ? [skill] : [] };
|
|
2017
|
+
}
|
|
2018
|
+
const skills = [];
|
|
2019
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
2020
|
+
if (!entry.isDirectory()) continue;
|
|
2021
|
+
const skill = readLocalSkill(resolve(dir, entry.name), `skills/${entry.name}`);
|
|
2022
|
+
if (skill) skills.push(skill);
|
|
2023
|
+
}
|
|
2024
|
+
if (skills.length > 0) {
|
|
2025
|
+
onProgress?.(`Found ${skills.length} skill(s)`);
|
|
2026
|
+
return { skills };
|
|
2027
|
+
}
|
|
2028
|
+
const content = await $fetch(`https://gitlab.com/${owner}/${repo}/-/raw/${ref}/SKILL.md`, { responseType: "text" }).catch(() => null);
|
|
2029
|
+
if (content) {
|
|
2030
|
+
const fm = parseSkillFrontmatterName(content);
|
|
2031
|
+
return { skills: [{
|
|
2032
|
+
name: fm.name || repo,
|
|
2033
|
+
description: fm.description || "",
|
|
2034
|
+
path: "",
|
|
2035
|
+
content,
|
|
2036
|
+
files: []
|
|
2037
|
+
}] };
|
|
2038
|
+
}
|
|
2039
|
+
return { skills: [] };
|
|
2040
|
+
} catch {
|
|
2041
|
+
return { skills: [] };
|
|
2042
|
+
} finally {
|
|
2043
|
+
rmSync(tempDir, {
|
|
2044
|
+
recursive: true,
|
|
2045
|
+
force: true
|
|
2046
|
+
});
|
|
1844
2047
|
}
|
|
1845
|
-
if (!gitDocsUrl && !readmeUrl && !llmsUrl) return null;
|
|
1846
|
-
return {
|
|
1847
|
-
name: repo,
|
|
1848
|
-
version: latestRelease ? version : void 0,
|
|
1849
|
-
releasedAt,
|
|
1850
|
-
description,
|
|
1851
|
-
repoUrl,
|
|
1852
|
-
docsUrl: homepage,
|
|
1853
|
-
gitDocsUrl,
|
|
1854
|
-
gitRef,
|
|
1855
|
-
gitDocsFallback: gitDocs?.fallback,
|
|
1856
|
-
readmeUrl: readmeUrl ?? void 0,
|
|
1857
|
-
llmsUrl
|
|
1858
|
-
};
|
|
1859
2048
|
}
|
|
1860
2049
|
async function searchNpmPackages(query, size = 5) {
|
|
1861
2050
|
const data = await $fetch(`https://registry.npmjs.org/-/v1/search?text=${encodeURIComponent(query)}&size=${size}`).catch(() => null);
|
|
@@ -2198,7 +2387,7 @@ async function fetchPkgDist(name, version) {
|
|
|
2198
2387
|
if (!data) return null;
|
|
2199
2388
|
const tarballUrl = data.dist?.tarball;
|
|
2200
2389
|
if (!tarballUrl) return null;
|
|
2201
|
-
const tarballRes = await fetch(tarballUrl, { headers: { "User-Agent":
|
|
2390
|
+
const tarballRes = await fetch(tarballUrl, { headers: { "User-Agent": SKILLD_USER_AGENT } }).catch(() => null);
|
|
2202
2391
|
if (!tarballRes?.ok || !tarballRes.body) return null;
|
|
2203
2392
|
mkdirSync(pkgDir, { recursive: true });
|
|
2204
2393
|
const tmpTarball = join(cacheDir, "_pkg.tgz");
|
|
@@ -2267,6 +2456,6 @@ function getInstalledSkillVersion(skillDir) {
|
|
|
2267
2456
|
if (!existsSync(skillPath)) return null;
|
|
2268
2457
|
return readFileSync(skillPath, "utf-8").match(/^version:\s*"?([^"\n]+)"?/m)?.[1] || null;
|
|
2269
2458
|
}
|
|
2270
|
-
export {
|
|
2459
|
+
export { fetchText as $, filterFrameworkDocs as A, fetchGitHubIssues as B, toCrawlPattern as C, fetchGitHubRepoMeta as D, fetchGitDocs as E, extractSections as F, compareSemver as G, generateIssueIndex as H, fetchLlmsTxt as I, isPrerelease as J, fetchReleaseNotes as K, fetchLlmsUrl as L, resolveGitHubRepo as M, validateGitDocsWithLlms as N, fetchReadme as O, downloadLlmsDocs as P, fetchGitHubRaw as Q, normalizeLlmsLinks as R, fetchCrawledDocs as S, MIN_GIT_DOCS as T, isGhAvailable as U, formatIssueAsMarkdown as V, fetchBlogReleases as W, $fetch as X, parseSemver as Y, extractBranchHint as Z, resolveEntryFiles as _, getInstalledSkillVersion as a, formatDiscussionAsMarkdown as b, readLocalPackageInfo as c, resolvePackageDocs as d, isGitHubRepoUrl as et, resolvePackageDocsWithAttempts as f, parseSkillFrontmatterName as g, parseGitSkillInput as h, fetchPkgDist as i, verifyUrl as it, isShallowGitDocs as j, fetchReadmeContent as k, resolveInstalledVersion as l, fetchGitSkills as m, fetchNpmPackage as n, parseGitHubUrl as nt, parseVersionSpecifier as o, searchNpmPackages as p, generateReleaseIndex as q, fetchNpmRegistryMeta as r, parsePackageSpec as rt, readLocalDependencies as s, fetchLatestVersion as t, normalizeRepoUrl as tt, resolveLocalPackageDocs as u, generateDocsIndex as v, resolveCrateDocsWithAttempts as w, generateDiscussionIndex as x, fetchGitHubDiscussions as y, parseMarkdownLinks as z };
|
|
2271
2460
|
|
|
2272
2461
|
//# sourceMappingURL=sources.mjs.map
|