skilld 1.6.2 → 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README.md +29 -20
  2. package/dist/_chunks/agent.mjs +2 -1
  3. package/dist/_chunks/agent.mjs.map +1 -1
  4. package/dist/_chunks/assemble.mjs +1 -1
  5. package/dist/_chunks/author-group.mjs +17 -0
  6. package/dist/_chunks/author-group.mjs.map +1 -0
  7. package/dist/_chunks/author.mjs +8 -6
  8. package/dist/_chunks/author.mjs.map +1 -1
  9. package/dist/_chunks/cache.mjs +1 -1
  10. package/dist/_chunks/cache2.mjs +1 -1
  11. package/dist/_chunks/cli-helpers.mjs +3 -119
  12. package/dist/_chunks/cli-helpers.mjs.map +1 -1
  13. package/dist/_chunks/config.mjs +119 -27
  14. package/dist/_chunks/config.mjs.map +1 -1
  15. package/dist/_chunks/core.mjs +1 -1
  16. package/dist/_chunks/embedding-cache2.mjs +1 -1
  17. package/dist/_chunks/index.d.mts.map +1 -1
  18. package/dist/_chunks/index3.d.mts +79 -78
  19. package/dist/_chunks/index3.d.mts.map +1 -1
  20. package/dist/_chunks/install.mjs +85 -535
  21. package/dist/_chunks/install.mjs.map +1 -1
  22. package/dist/_chunks/install2.mjs +554 -0
  23. package/dist/_chunks/install2.mjs.map +1 -0
  24. package/dist/_chunks/lockfile.mjs +1 -0
  25. package/dist/_chunks/lockfile.mjs.map +1 -1
  26. package/dist/_chunks/package-registry.mjs +465 -0
  27. package/dist/_chunks/package-registry.mjs.map +1 -0
  28. package/dist/_chunks/prefix.mjs +108 -0
  29. package/dist/_chunks/prefix.mjs.map +1 -0
  30. package/dist/_chunks/prepare.mjs +6 -2
  31. package/dist/_chunks/prepare.mjs.map +1 -1
  32. package/dist/_chunks/prepare2.mjs +1 -1
  33. package/dist/_chunks/prompts.mjs +5 -99
  34. package/dist/_chunks/prompts.mjs.map +1 -1
  35. package/dist/_chunks/search-helpers.mjs +99 -0
  36. package/dist/_chunks/search-helpers.mjs.map +1 -0
  37. package/dist/_chunks/search-interactive.mjs +1 -1
  38. package/dist/_chunks/search-interactive.mjs.map +1 -1
  39. package/dist/_chunks/search.mjs +219 -1
  40. package/dist/_chunks/search.mjs.map +1 -0
  41. package/dist/_chunks/shared.mjs +1 -463
  42. package/dist/_chunks/shared.mjs.map +1 -1
  43. package/dist/_chunks/skills.mjs +1 -1
  44. package/dist/_chunks/sources.mjs +1177 -988
  45. package/dist/_chunks/sources.mjs.map +1 -1
  46. package/dist/_chunks/sync-registry.mjs +59 -0
  47. package/dist/_chunks/sync-registry.mjs.map +1 -0
  48. package/dist/_chunks/sync-shared2.mjs +10 -7
  49. package/dist/_chunks/sync-shared2.mjs.map +1 -1
  50. package/dist/_chunks/sync.mjs +208 -99
  51. package/dist/_chunks/sync.mjs.map +1 -1
  52. package/dist/_chunks/sync2.mjs +1 -1
  53. package/dist/_chunks/uninstall.mjs +3 -2
  54. package/dist/_chunks/uninstall.mjs.map +1 -1
  55. package/dist/_chunks/upload.mjs +152 -0
  56. package/dist/_chunks/upload.mjs.map +1 -0
  57. package/dist/_chunks/validate.mjs +1 -1
  58. package/dist/_chunks/version.mjs +30 -0
  59. package/dist/_chunks/version.mjs.map +1 -0
  60. package/dist/_chunks/wizard.mjs +2 -1
  61. package/dist/_chunks/wizard.mjs.map +1 -1
  62. package/dist/agent/index.mjs +2 -1
  63. package/dist/cache/index.mjs +1 -1
  64. package/dist/cli.mjs +48 -20
  65. package/dist/cli.mjs.map +1 -1
  66. package/dist/index.d.mts +1 -1
  67. package/dist/index.mjs +2 -2
  68. package/dist/sources/index.d.mts +2 -2
  69. package/dist/sources/index.mjs +3 -3
  70. package/dist/types.d.mts +1 -1
  71. package/package.json +11 -12
  72. package/dist/_chunks/search2.mjs +0 -310
  73. package/dist/_chunks/search2.mjs.map +0 -1
@@ -1,8 +1,9 @@
1
- import { o as getCacheDir } from "./config.mjs";
1
+ import { t as getCacheDir } from "./version.mjs";
2
2
  import { i as readPackageJsonSafe } from "./package-json.mjs";
3
3
  import { t as yamlEscape } from "./yaml.mjs";
4
4
  import { i as parseFrontmatter, n as extractLinks, r as extractTitle, t as extractDescription } from "./markdown.mjs";
5
- import { c as getBlogPreset, l as getCrawlUrl, r as mapInsert, u as getDocOverride } from "./shared.mjs";
5
+ import { n as getCrawlUrl, r as getDocOverride, t as getBlogPreset } from "./package-registry.mjs";
6
+ import { r as mapInsert } from "./shared.mjs";
6
7
  import { tmpdir } from "node:os";
7
8
  import { basename, dirname, join, resolve } from "pathe";
8
9
  import { createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, rmSync } from "node:fs";
@@ -10,10 +11,10 @@ import { htmlToMarkdown } from "mdream";
10
11
  import pLimit from "p-limit";
11
12
  import { spawnSync } from "node:child_process";
12
13
  import { ofetch } from "ofetch";
14
+ import { fileURLToPath, pathToFileURL } from "node:url";
13
15
  import { crawlAndGenerate } from "@mdream/crawl";
14
16
  import { glob } from "tinyglobby";
15
17
  import { downloadTemplate } from "giget";
16
- import { fileURLToPath, pathToFileURL } from "node:url";
17
18
  import { Writable } from "node:stream";
18
19
  import { resolvePathSync } from "mlly";
19
20
  const BOT_USERS = new Set([
@@ -110,12 +111,36 @@ async function ghApiPaginated(endpoint) {
110
111
  }
111
112
  return results;
112
113
  }
114
+ const SKILLD_USER_AGENT = "skilld/1.0 (+https://github.com/harlan-zw/skilld)";
113
115
  const $fetch = ofetch.create({
114
116
  retry: 3,
115
- retryDelay: 500,
117
+ retryDelay: 1e3,
118
+ retryStatusCodes: [
119
+ 408,
120
+ 429,
121
+ 500,
122
+ 502,
123
+ 503,
124
+ 504
125
+ ],
116
126
  timeout: 15e3,
117
- headers: { "User-Agent": "skilld/1.0" }
127
+ headers: { "User-Agent": SKILLD_USER_AGENT }
118
128
  });
129
+ function createRateLimitedRunner(intervalMs) {
130
+ let queue = Promise.resolve();
131
+ let lastRunAt = 0;
132
+ return async function runRateLimited(task) {
133
+ const run = async () => {
134
+ const waitMs = intervalMs - (Date.now() - lastRunAt);
135
+ if (waitMs > 0) await new Promise((resolve) => setTimeout(resolve, waitMs));
136
+ lastRunAt = Date.now();
137
+ return task();
138
+ };
139
+ const request = queue.then(run, run);
140
+ queue = request.then(() => void 0, () => void 0);
141
+ return request;
142
+ };
143
+ }
119
144
  async function fetchText(url) {
120
145
  return $fetch(url, { responseType: "text" }).catch(() => null);
121
146
  }
@@ -175,6 +200,20 @@ function isGitHubRepoUrl(url) {
175
200
  return false;
176
201
  }
177
202
  }
203
+ function isLikelyCodeHostUrl(url) {
204
+ if (!url) return false;
205
+ try {
206
+ const parsed = new URL(url);
207
+ return [
208
+ "github.com",
209
+ "www.github.com",
210
+ "gitlab.com",
211
+ "www.gitlab.com"
212
+ ].includes(parsed.hostname);
213
+ } catch {
214
+ return false;
215
+ }
216
+ }
178
217
  function parseGitHubUrl(url) {
179
218
  const match = url.match(/github\.com\/([^/]+)\/([^/]+?)(?:\.git)?(?:[/#]|$)/);
180
219
  if (!match) return null;
@@ -456,99 +495,6 @@ async function fetchBlogReleases(packageName, installedVersion) {
456
495
  content: formatBlogRelease(r)
457
496
  }));
458
497
  }
459
- async function fetchCrawledDocs(url, onProgress, maxPages = 200) {
460
- const outputDir = join(tmpdir(), "skilld-crawl", Date.now().toString());
461
- onProgress?.(`Crawling ${url}`);
462
- const userLang = getUserLang();
463
- const foreignUrls = /* @__PURE__ */ new Set();
464
- const doCrawl = () => crawlAndGenerate({
465
- urls: [url],
466
- outputDir,
467
- driver: "http",
468
- generateLlmsTxt: false,
469
- generateIndividualMd: true,
470
- maxRequestsPerCrawl: maxPages,
471
- onPage: (page) => {
472
- const lang = extractHtmlLang(page.html);
473
- if (lang && !lang.startsWith("en") && !lang.startsWith(userLang)) foreignUrls.add(page.url);
474
- }
475
- }, (progress) => {
476
- if (progress.crawling.status === "processing" && progress.crawling.total > 0) onProgress?.(`Crawling ${progress.crawling.processed}/${progress.crawling.total} pages`);
477
- });
478
- let results = await doCrawl().catch((err) => {
479
- onProgress?.(`Crawl failed: ${err?.message || err}`);
480
- return [];
481
- });
482
- if (results.length === 0) {
483
- onProgress?.("Retrying crawl");
484
- results = await doCrawl().catch(() => []);
485
- }
486
- rmSync(outputDir, {
487
- recursive: true,
488
- force: true
489
- });
490
- const docs = [];
491
- let localeFiltered = 0;
492
- for (const result of results) {
493
- if (!result.success || !result.content) continue;
494
- if (foreignUrls.has(result.url)) {
495
- localeFiltered++;
496
- continue;
497
- }
498
- const segments = (new URL(result.url).pathname.replace(/\/$/, "") || "/index").split("/").filter(Boolean);
499
- if (isForeignPathPrefix(segments[0], userLang)) {
500
- localeFiltered++;
501
- continue;
502
- }
503
- const path = `docs/${segments.join("/")}.md`;
504
- docs.push({
505
- path,
506
- content: result.content
507
- });
508
- }
509
- if (localeFiltered > 0) onProgress?.(`Filtered ${localeFiltered} foreign locale pages`);
510
- onProgress?.(`Crawled ${docs.length} pages`);
511
- return docs;
512
- }
513
- const HTML_LANG_RE = /<html[^>]*\slang=["']([^"']+)["']/i;
514
- function extractHtmlLang(html) {
515
- return HTML_LANG_RE.exec(html)?.[1]?.toLowerCase();
516
- }
517
- const LOCALE_CODES = new Set([
518
- "ar",
519
- "de",
520
- "es",
521
- "fr",
522
- "id",
523
- "it",
524
- "ja",
525
- "ko",
526
- "nl",
527
- "pl",
528
- "pt",
529
- "pt-br",
530
- "ru",
531
- "th",
532
- "tr",
533
- "uk",
534
- "vi",
535
- "zh",
536
- "zh-cn",
537
- "zh-tw"
538
- ]);
539
- function isForeignPathPrefix(segment, userLang) {
540
- if (!segment) return false;
541
- const lower = segment.toLowerCase();
542
- if (lower === "en" || lower.startsWith(userLang)) return false;
543
- return LOCALE_CODES.has(lower);
544
- }
545
- function getUserLang() {
546
- const code = (process.env.LC_ALL || process.env.LANG || process.env.LANGUAGE || "").split(/[_.:-]/)[0]?.toLowerCase() || "";
547
- return code.length >= 2 ? code.slice(0, 2) : "en";
548
- }
549
- function toCrawlPattern(docsUrl) {
550
- return `${docsUrl.replace(/\/+$/, "")}/**`;
551
- }
552
498
  let _ghAvailable;
553
499
  function isGhAvailable() {
554
500
  if (_ghAvailable !== void 0) return _ghAvailable;
@@ -870,992 +816,1235 @@ function generateIssueIndex(issues) {
870
816
  }
871
817
  return sections.join("\n");
872
818
  }
873
- const HIGH_VALUE_CATEGORIES = new Set([
874
- "q&a",
875
- "help",
876
- "troubleshooting",
877
- "support"
878
- ]);
879
- const LOW_VALUE_CATEGORIES = new Set([
880
- "show and tell",
881
- "ideas",
882
- "polls"
883
- ]);
884
- const TITLE_NOISE_RE = /looking .*(?:developer|engineer|freelanc)|hiring|job post|guide me to (?:complete|finish|build)|help me (?:complete|finish|build)|seeking .* tutorial|recommend.* course/i;
885
- const MIN_DISCUSSION_SCORE = 3;
886
- function scoreComment(c) {
887
- return (c.isMaintainer ? 3 : 1) * (hasCodeBlock(c.body) ? 2 : 1) * (1 + c.reactions);
819
+ async function fetchLlmsUrl(docsUrl) {
820
+ const llmsUrl = `${new URL(docsUrl).origin}/llms.txt`;
821
+ if (await verifyUrl(llmsUrl)) return llmsUrl;
822
+ return null;
888
823
  }
889
- function scoreDiscussion(d) {
890
- if (TITLE_NOISE_RE.test(d.title)) return -1;
891
- let score = 0;
892
- if (d.isMaintainer) score += 3;
893
- if (hasCodeBlock([
894
- d.body,
895
- d.answer || "",
896
- ...d.topComments.map((c) => c.body)
897
- ].join("\n"))) score += 3;
898
- score += Math.min(d.upvoteCount, 5);
899
- if (d.answer) {
900
- score += 2;
901
- if (d.answer.length > 100) score += 1;
902
- }
903
- if (d.topComments.some((c) => c.isMaintainer)) score += 2;
904
- if (d.topComments.some((c) => c.reactions > 0)) score += 1;
905
- return score;
824
+ async function fetchLlmsTxt(url) {
825
+ const content = await fetchText(url);
826
+ if (!content || content.length < 50) return null;
827
+ return {
828
+ raw: content,
829
+ links: parseMarkdownLinks(content)
830
+ };
906
831
  }
907
- async function fetchGitHubDiscussions(owner, repo, limit = 20, releasedAt, fromDate) {
908
- if (!isGhAvailable()) return [];
909
- if (!fromDate && releasedAt) {
910
- const cutoff = new Date(releasedAt);
911
- cutoff.setMonth(cutoff.getMonth() + 6);
912
- if (cutoff < /* @__PURE__ */ new Date()) return [];
913
- }
832
+ function parseMarkdownLinks(content) {
833
+ return extractLinks(content).filter((l) => l.url.endsWith(".md"));
834
+ }
835
+ function isSafeUrl(url) {
914
836
  try {
915
- const { stdout: result } = spawnSync("gh", [
916
- "api",
917
- "graphql",
918
- "-f",
919
- `query=${`query($owner: String!, $repo: String!) { repository(owner: $owner, name: $repo) { discussions(first: ${Math.min(limit * 3, 80)}, orderBy: {field: CREATED_AT, direction: DESC}) { nodes { number title body category { name } createdAt url upvoteCount comments(first: 10) { totalCount nodes { body author { login } authorAssociation reactions { totalCount } } } answer { body author { login } authorAssociation } author { login } authorAssociation } } } }`}`,
920
- "-f",
921
- `owner=${owner}`,
922
- "-f",
923
- `repo=${repo}`
924
- ], {
925
- encoding: "utf-8",
926
- maxBuffer: 10 * 1024 * 1024
927
- });
928
- if (!result) return [];
929
- const nodes = JSON.parse(result)?.data?.repository?.discussions?.nodes;
930
- if (!Array.isArray(nodes)) return [];
931
- const fromTs = fromDate ? new Date(fromDate).getTime() : null;
932
- return nodes.filter((d) => d.author && !BOT_USERS.has(d.author.login)).filter((d) => {
933
- const cat = (d.category?.name || "").toLowerCase();
934
- return !LOW_VALUE_CATEGORIES.has(cat);
935
- }).filter((d) => !fromTs || new Date(d.createdAt).getTime() >= fromTs).map((d) => {
936
- let answer;
937
- if (d.answer?.body) {
938
- const isMaintainer = [
939
- "OWNER",
940
- "MEMBER",
941
- "COLLABORATOR"
942
- ].includes(d.answer.authorAssociation);
943
- const author = d.answer.author?.login;
944
- answer = `${isMaintainer && author ? `**@${author}** [maintainer]:\n\n` : ""}${d.answer.body}`;
945
- }
946
- const comments = (d.comments?.nodes || []).filter((c) => c.author && !BOT_USERS.has(c.author.login)).filter((c) => !COMMENT_NOISE_RE.test((c.body || "").trim())).map((c) => {
947
- const isMaintainer = [
948
- "OWNER",
949
- "MEMBER",
950
- "COLLABORATOR"
951
- ].includes(c.authorAssociation);
952
- return {
953
- body: c.body || "",
954
- author: c.author.login,
955
- reactions: c.reactions?.totalCount || 0,
956
- isMaintainer
957
- };
958
- }).sort((a, b) => scoreComment(b) - scoreComment(a)).slice(0, 3);
959
- return {
960
- number: d.number,
961
- title: d.title,
962
- body: d.body || "",
963
- category: d.category?.name || "",
964
- createdAt: d.createdAt,
965
- url: d.url,
966
- upvoteCount: d.upvoteCount || 0,
967
- comments: d.comments?.totalCount || 0,
968
- isMaintainer: [
969
- "OWNER",
970
- "MEMBER",
971
- "COLLABORATOR"
972
- ].includes(d.authorAssociation),
973
- answer,
974
- topComments: comments
975
- };
976
- }).map((d) => ({
977
- d,
978
- score: scoreDiscussion(d)
979
- })).filter(({ score }) => score >= MIN_DISCUSSION_SCORE).sort((a, b) => {
980
- const aHigh = HIGH_VALUE_CATEGORIES.has(a.d.category.toLowerCase()) ? 1 : 0;
981
- const bHigh = HIGH_VALUE_CATEGORIES.has(b.d.category.toLowerCase()) ? 1 : 0;
982
- if (aHigh !== bHigh) return bHigh - aHigh;
983
- return b.score - a.score;
984
- }).slice(0, limit).map(({ d }) => d);
837
+ const parsed = new URL(url);
838
+ if (parsed.protocol !== "https:") return false;
839
+ const host = parsed.hostname;
840
+ if (host === "localhost" || host === "0.0.0.0" || host === "[::1]") return false;
841
+ if (/^(?:127\.|10\.|172\.(?:1[6-9]|2\d|3[01])\.|192\.168\.|169\.254\.)/.test(host)) return false;
842
+ if (/^\[(?:f[cd]|fe[89ab]|::ffff:)/i.test(host)) return false;
843
+ return true;
985
844
  } catch {
986
- return [];
845
+ return false;
987
846
  }
988
847
  }
989
- function formatDiscussionAsMarkdown(d) {
990
- const fm = buildFrontmatter({
991
- number: d.number,
992
- title: d.title,
993
- category: d.category,
994
- created: isoDate(d.createdAt),
995
- url: d.url,
996
- upvotes: d.upvoteCount,
997
- comments: d.comments,
998
- answered: !!d.answer
999
- });
1000
- const bodyLimit = d.upvoteCount >= 5 ? 1500 : 800;
1001
- const lines = [
1002
- fm,
1003
- "",
1004
- `# ${d.title}`
1005
- ];
1006
- if (d.body) lines.push("", truncateBody(d.body, bodyLimit));
1007
- if (d.answer) lines.push("", "---", "", "## Accepted Answer", "", truncateBody(d.answer, 1e3));
1008
- else if (d.topComments.length > 0) {
1009
- lines.push("", "---", "", "## Top Comments");
1010
- for (const c of d.topComments) {
1011
- const reactions = c.reactions > 0 ? ` (+${c.reactions})` : "";
1012
- const maintainer = c.isMaintainer ? " [maintainer]" : "";
1013
- lines.push("", `**@${c.author}**${maintainer}${reactions}:`, "", truncateBody(c.body, 600));
1014
- }
1015
- }
1016
- return lines.join("\n");
848
+ async function downloadLlmsDocs(llmsContent, baseUrl, onProgress) {
849
+ const limit = pLimit(5);
850
+ let completed = 0;
851
+ return (await Promise.all(llmsContent.links.map((link) => limit(async () => {
852
+ const url = link.url.startsWith("http") ? link.url : `${baseUrl.replace(/\/$/, "")}${link.url.startsWith("/") ? "" : "/"}${link.url}`;
853
+ if (!isSafeUrl(url)) return null;
854
+ const content = await fetchText(url);
855
+ onProgress?.(link.url, ++completed, llmsContent.links.length);
856
+ if (content && content.length > 100) return {
857
+ url: link.url.startsWith("http") ? new URL(link.url).pathname : link.url,
858
+ title: link.title,
859
+ content
860
+ };
861
+ return null;
862
+ })))).filter((d) => d !== null);
1017
863
  }
1018
- function generateDiscussionIndex(discussions) {
1019
- const byCategory = /* @__PURE__ */ new Map();
1020
- for (const d of discussions) mapInsert(byCategory, d.category || "Uncategorized", () => []).push(d);
1021
- const answered = discussions.filter((d) => d.answer).length;
1022
- const sections = [
1023
- [
1024
- "---",
1025
- `total: ${discussions.length}`,
1026
- `answered: ${answered}`,
1027
- "---"
1028
- ].join("\n"),
1029
- "",
1030
- "# Discussions Index",
1031
- ""
1032
- ];
1033
- const cats = [...byCategory.keys()].sort((a, b) => {
1034
- return (HIGH_VALUE_CATEGORIES.has(a.toLowerCase()) ? 0 : 1) - (HIGH_VALUE_CATEGORIES.has(b.toLowerCase()) ? 0 : 1) || a.localeCompare(b);
1035
- });
1036
- for (const cat of cats) {
1037
- const group = byCategory.get(cat);
1038
- sections.push(`## ${cat} (${group.length})`, "");
1039
- for (const d of group) {
1040
- const upvotes = d.upvoteCount > 0 ? ` (+${d.upvoteCount})` : "";
1041
- const answered = d.answer ? " [answered]" : "";
1042
- const date = isoDate(d.createdAt);
1043
- sections.push(`- [#${d.number}](./discussion-${d.number}.md): ${d.title}${upvotes}${answered} (${date})`);
1044
- }
1045
- sections.push("");
864
+ function normalizeLlmsLinks(content, baseUrl) {
865
+ let normalized = content;
866
+ if (baseUrl) {
867
+ const escaped = baseUrl.replace(/\/$/, "").replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
868
+ normalized = normalized.replace(new RegExp(`\\]\\(${escaped}(/[^)]+\\.md)\\)`, "g"), "](./docs$1)");
1046
869
  }
1047
- return sections.join("\n");
870
+ normalized = normalized.replace(/\]\(\/([^)]+\.md)\)/g, "](./docs/$1)");
871
+ return normalized;
1048
872
  }
1049
- function generateDocsIndex(docs) {
1050
- const docFiles = docs.filter((d) => d.path.startsWith("docs/") && d.path.endsWith(".md") && !d.path.endsWith("_INDEX.md")).sort((a, b) => a.path.localeCompare(b.path));
1051
- if (docFiles.length === 0) return "";
1052
- const rootFiles = [];
1053
- const byDir = /* @__PURE__ */ new Map();
1054
- for (const doc of docFiles) {
1055
- const rel = doc.path.slice(5);
1056
- const dir = rel.includes("/") ? rel.slice(0, rel.lastIndexOf("/")) : "";
1057
- if (!dir) rootFiles.push(doc);
1058
- else {
1059
- const list = byDir.get(dir);
1060
- if (list) list.push(doc);
1061
- else byDir.set(dir, [doc]);
873
+ function extractSections(content, patterns) {
874
+ const sections = [];
875
+ const parts = content.split(/\n---\n/);
876
+ for (const part of parts) {
877
+ const urlMatch = part.match(/^url: *(\S.*)$/m);
878
+ if (!urlMatch) continue;
879
+ const url = urlMatch[1];
880
+ if (patterns.some((p) => url.includes(p))) {
881
+ const contentStart = part.indexOf("\n", part.indexOf("url:"));
882
+ if (contentStart > -1) sections.push(part.slice(contentStart + 1));
1062
883
  }
1063
884
  }
1064
- const sections = [
1065
- "---",
1066
- `total: ${docFiles.length}`,
1067
- "---",
1068
- "",
1069
- "# Docs Index",
1070
- ""
1071
- ];
1072
- for (const file of rootFiles) {
1073
- const rel = file.path.slice(5);
1074
- const title = extractTitle(file.content) || rel.replace(/\.md$/, "");
1075
- const desc = extractDescription(file.content);
1076
- const descPart = desc ? `: ${desc}` : "";
1077
- sections.push(`- [${title}](./${rel})${descPart}`);
885
+ if (sections.length === 0) return null;
886
+ return sections.join("\n\n---\n\n");
887
+ }
888
+ const MIN_GIT_DOCS = 5;
889
+ const isShallowGitDocs = (n) => n > 0 && n < 5;
890
+ async function listFilesAtRef(owner, repo, ref) {
891
+ if (!isKnownPrivateRepo(owner, repo)) {
892
+ const data = await $fetch(`https://ungh.cc/repos/${owner}/${repo}/files/${ref}`).catch(() => null);
893
+ if (data?.files?.length) return data.files.map((f) => f.path);
1078
894
  }
1079
- if (rootFiles.length > 0) sections.push("");
1080
- for (const [dir, files] of byDir) {
1081
- sections.push(`## ${dir} (${files.length})`, "");
1082
- for (const file of files) {
1083
- const rel = file.path.slice(5);
1084
- const title = extractTitle(file.content) || rel.replace(/\.md$/, "").split("/").pop();
1085
- const desc = extractDescription(file.content);
1086
- const descPart = desc ? `: ${desc}` : "";
1087
- sections.push(`- [${title}](./${rel})${descPart}`);
1088
- }
1089
- sections.push("");
895
+ const tree = await ghApi(`repos/${owner}/${repo}/git/trees/${ref}?recursive=1`);
896
+ if (tree?.tree?.length) {
897
+ markRepoPrivate(owner, repo);
898
+ return tree.tree.map((f) => f.path);
1090
899
  }
1091
- return sections.join("\n");
900
+ return [];
1092
901
  }
1093
- const SKIP_DIRS = [
1094
- "node_modules",
1095
- "_vendor",
1096
- "__tests__",
1097
- "__mocks__",
1098
- "__fixtures__",
1099
- "test",
1100
- "tests",
1101
- "fixture",
1102
- "fixtures",
1103
- "locales",
1104
- "locale",
1105
- "i18n",
1106
- ".git"
1107
- ];
1108
- const SKIP_PATTERNS = [
1109
- "*.min.*",
1110
- "*.prod.*",
1111
- "*.global.*",
1112
- "*.browser.*",
1113
- "*.map",
1114
- "*.map.js",
1115
- "CHANGELOG*",
1116
- "LICENSE*",
1117
- "README*"
1118
- ];
1119
- const MAX_FILE_SIZE = 500 * 1024;
1120
- async function resolveEntryFiles(packageDir) {
1121
- if (!existsSync(join(packageDir, "package.json"))) return [];
1122
- const files = await glob(["**/*.d.{ts,mts,cts}"], {
1123
- cwd: packageDir,
1124
- ignore: [...SKIP_DIRS.map((d) => `**/${d}/**`), ...SKIP_PATTERNS],
1125
- absolute: false,
1126
- expandDirectories: false
1127
- });
1128
- const entries = [];
1129
- for (const file of files) {
1130
- const absPath = join(packageDir, file);
1131
- let content;
1132
- try {
1133
- content = readFileSync(absPath, "utf-8");
1134
- } catch {
1135
- continue;
902
+ async function findGitTag(owner, repo, version, packageName, branchHint) {
903
+ const candidates = [`v${version}`, version];
904
+ if (packageName) candidates.push(`${packageName}@${version}`);
905
+ for (const tag of candidates) {
906
+ const files = await listFilesAtRef(owner, repo, tag);
907
+ if (files.length > 0) return {
908
+ ref: tag,
909
+ files
910
+ };
911
+ }
912
+ if (packageName) {
913
+ const latestTag = await findLatestReleaseTag(owner, repo, packageName);
914
+ if (latestTag) {
915
+ const files = await listFilesAtRef(owner, repo, latestTag);
916
+ if (files.length > 0) return {
917
+ ref: latestTag,
918
+ files
919
+ };
1136
920
  }
1137
- if (content.length > MAX_FILE_SIZE) continue;
1138
- entries.push({
1139
- path: file,
1140
- content,
1141
- type: "types"
1142
- });
1143
921
  }
1144
- return entries;
1145
- }
1146
- function parseGitSkillInput(input) {
1147
- const trimmed = input.trim();
1148
- if (trimmed.startsWith("@")) return null;
1149
- if (trimmed.startsWith("./") || trimmed.startsWith("../") || trimmed.startsWith("/") || trimmed.startsWith("~")) return {
1150
- type: "local",
1151
- localPath: trimmed.startsWith("~") ? resolve(process.env.HOME || "", trimmed.slice(1)) : resolve(trimmed)
1152
- };
1153
- if (trimmed.startsWith("git@")) {
1154
- const gh = parseGitHubUrl(normalizeRepoUrl(trimmed));
1155
- if (gh) return {
1156
- type: "github",
1157
- owner: gh.owner,
1158
- repo: gh.repo
922
+ const branches = branchHint ? [branchHint, ...["main", "master"].filter((b) => b !== branchHint)] : ["main", "master"];
923
+ for (const branch of branches) {
924
+ const files = await listFilesAtRef(owner, repo, branch);
925
+ if (files.length > 0) return {
926
+ ref: branch,
927
+ files,
928
+ fallback: true
1159
929
  };
1160
- return null;
1161
930
  }
1162
- if (trimmed.startsWith("https://") || trimmed.startsWith("http://")) return parseGitUrl(trimmed);
1163
- if (/^[\w.-]+\/[\w.-]+$/.test(trimmed)) return {
1164
- type: "github",
1165
- owner: trimmed.split("/")[0],
1166
- repo: trimmed.split("/")[1]
1167
- };
1168
931
  return null;
1169
932
  }
1170
- function parseGitUrl(url) {
1171
- try {
1172
- const parsed = new URL(url);
1173
- if (parsed.hostname === "github.com" || parsed.hostname === "www.github.com") {
1174
- const parts = parsed.pathname.replace(/^\//, "").replace(/\.git$/, "").split("/");
1175
- const owner = parts[0];
1176
- const repo = parts[1];
1177
- if (!owner || !repo) return null;
1178
- if (parts[2] === "tree" && parts.length >= 4) return {
1179
- type: "github",
1180
- owner,
1181
- repo,
1182
- ref: parts[3],
1183
- skillPath: parts.length > 4 ? parts.slice(4).join("/") : void 0
1184
- };
1185
- return {
1186
- type: "github",
1187
- owner,
1188
- repo
1189
- };
1190
- }
1191
- if (parsed.hostname === "gitlab.com") {
1192
- const parts = parsed.pathname.replace(/^\//, "").replace(/\.git$/, "").split("/");
1193
- const owner = parts[0];
1194
- const repo = parts[1];
1195
- if (!owner || !repo) return null;
1196
- return {
1197
- type: "gitlab",
1198
- owner,
1199
- repo
1200
- };
1201
- }
1202
- return null;
1203
- } catch {
1204
- return null;
933
+ async function fetchUnghReleases(owner, repo) {
934
+ if (!isKnownPrivateRepo(owner, repo)) {
935
+ const data = await $fetch(`https://ungh.cc/repos/${owner}/${repo}/releases`).catch(() => null);
936
+ if (data?.releases?.length) return data.releases;
937
+ }
938
+ const raw = await ghApiPaginated(`repos/${owner}/${repo}/releases`);
939
+ if (raw.length > 0) {
940
+ markRepoPrivate(owner, repo);
941
+ return raw.map((r) => ({
942
+ tag: r.tag_name,
943
+ publishedAt: r.published_at
944
+ }));
1205
945
  }
946
+ return [];
1206
947
  }
1207
- function parseSkillFrontmatterName(content) {
1208
- const fm = parseFrontmatter(content);
1209
- return {
1210
- name: fm.name,
1211
- description: fm.description
1212
- };
948
+ async function findLatestReleaseTag(owner, repo, packageName) {
949
+ const prefix = `${packageName}@`;
950
+ return (await fetchUnghReleases(owner, repo)).find((r) => r.tag.startsWith(prefix))?.tag ?? null;
1213
951
  }
1214
- function collectFiles(dir, prefix = "") {
1215
- const files = [];
1216
- if (!existsSync(dir)) return files;
1217
- for (const entry of readdirSync(dir, { withFileTypes: true })) {
1218
- const relPath = prefix ? `${prefix}/${entry.name}` : entry.name;
1219
- const fullPath = resolve(dir, entry.name);
1220
- if (entry.isDirectory()) files.push(...collectFiles(fullPath, relPath));
1221
- else if (entry.isFile()) files.push({
1222
- path: relPath,
1223
- content: readFileSync(fullPath, "utf-8")
1224
- });
1225
- }
1226
- return files;
952
+ function filterDocFiles(files, pathPrefix) {
953
+ return files.filter((f) => f.startsWith(pathPrefix) && /\.(?:md|mdx)$/.test(f));
1227
954
  }
1228
- async function fetchGitSkills(source, onProgress) {
1229
- if (source.type === "local") return fetchLocalSkills(source);
1230
- if (source.type === "github") return fetchGitHubSkills(source, onProgress);
1231
- if (source.type === "gitlab") return fetchGitLabSkills(source, onProgress);
1232
- return { skills: [] };
955
+ const FRAMEWORK_NAMES = new Set([
956
+ "vue",
957
+ "react",
958
+ "solid",
959
+ "angular",
960
+ "svelte",
961
+ "preact",
962
+ "lit",
963
+ "qwik"
964
+ ]);
965
+ function filterFrameworkDocs(files, packageName) {
966
+ if (!packageName) return files;
967
+ const shortName = packageName.replace(/^@.*\//, "");
968
+ const targetFramework = [...FRAMEWORK_NAMES].find((fw) => shortName.includes(fw));
969
+ if (!targetFramework) return files;
970
+ const otherFrameworks = [...FRAMEWORK_NAMES].filter((fw) => fw !== targetFramework);
971
+ const excludePattern = new RegExp(`\\b(?:${otherFrameworks.join("|")})\\b`);
972
+ return files.filter((f) => !excludePattern.test(f));
1233
973
  }
1234
- function fetchLocalSkills(source) {
1235
- const base = source.localPath;
1236
- if (!existsSync(base)) return { skills: [] };
1237
- const skills = [];
1238
- const skillsDir = resolve(base, "skills");
1239
- if (existsSync(skillsDir)) for (const entry of readdirSync(skillsDir, { withFileTypes: true })) {
1240
- if (!entry.isDirectory()) continue;
1241
- const skill = readLocalSkill(resolve(skillsDir, entry.name), `skills/${entry.name}`);
1242
- if (skill) skills.push(skill);
974
+ const NOISE_PATTERNS = [
975
+ /^\.changeset\//,
976
+ /CHANGELOG\.md$/i,
977
+ /CONTRIBUTING\.md$/i,
978
+ /^\.github\//
979
+ ];
980
+ const EXCLUDE_DIRS = new Set([
981
+ "test",
982
+ "tests",
983
+ "__tests__",
984
+ "fixtures",
985
+ "fixture",
986
+ "examples",
987
+ "example",
988
+ "node_modules",
989
+ ".git",
990
+ "dist",
991
+ "build",
992
+ "coverage",
993
+ "e2e",
994
+ "spec",
995
+ "mocks",
996
+ "__mocks__"
997
+ ]);
998
+ const DOC_DIR_BONUS = new Set([
999
+ "docs",
1000
+ "documentation",
1001
+ "pages",
1002
+ "content",
1003
+ "website",
1004
+ "guide",
1005
+ "guides",
1006
+ "wiki",
1007
+ "manual",
1008
+ "api"
1009
+ ]);
1010
+ function hasExcludedDir(path) {
1011
+ return path.split("/").some((p) => EXCLUDE_DIRS.has(p.toLowerCase()));
1012
+ }
1013
+ function getPathDepth(path) {
1014
+ return path.split("/").filter(Boolean).length;
1015
+ }
1016
+ function hasDocDirBonus(path) {
1017
+ return path.split("/").some((p) => DOC_DIR_BONUS.has(p.toLowerCase()));
1018
+ }
1019
+ function scoreDocDir(dir, fileCount) {
1020
+ const depth = getPathDepth(dir) || 1;
1021
+ return fileCount * (hasDocDirBonus(dir) ? 1.5 : 1) / depth;
1022
+ }
1023
+ function discoverDocFiles(allFiles, packageName) {
1024
+ const mdFiles = allFiles.filter((f) => /\.(?:md|mdx)$/.test(f)).filter((f) => !NOISE_PATTERNS.some((p) => p.test(f))).filter((f) => f.includes("/"));
1025
+ if (packageName?.includes("/")) {
1026
+ const subPkgPrefix = `packages/${packageName.split("/").pop().toLowerCase()}/`;
1027
+ const subPkgFiles = mdFiles.filter((f) => f.startsWith(subPkgPrefix));
1028
+ if (subPkgFiles.length >= 3) return {
1029
+ files: subPkgFiles,
1030
+ prefix: subPkgPrefix
1031
+ };
1243
1032
  }
1244
- if (skills.length === 0) {
1245
- const skill = readLocalSkill(base, "");
1246
- if (skill) skills.push(skill);
1033
+ const docsGroups = /* @__PURE__ */ new Map();
1034
+ for (const file of mdFiles) {
1035
+ const docsIdx = file.lastIndexOf("/docs/");
1036
+ if (docsIdx === -1) continue;
1037
+ mapInsert(docsGroups, file.slice(0, docsIdx + 6), () => []).push(file);
1247
1038
  }
1248
- return { skills };
1039
+ if (docsGroups.size > 0) {
1040
+ const largest = [...docsGroups.entries()].sort((a, b) => b[1].length - a[1].length)[0];
1041
+ if (largest[1].length >= 3) {
1042
+ const fullPrefix = largest[0];
1043
+ const docsIdx = fullPrefix.lastIndexOf("docs/");
1044
+ const stripPrefix = docsIdx > 0 ? fullPrefix.slice(0, docsIdx) : "";
1045
+ return {
1046
+ files: largest[1],
1047
+ prefix: stripPrefix
1048
+ };
1049
+ }
1050
+ }
1051
+ const dirGroups = /* @__PURE__ */ new Map();
1052
+ for (const file of mdFiles) {
1053
+ if (hasExcludedDir(file)) continue;
1054
+ const lastSlash = file.lastIndexOf("/");
1055
+ if (lastSlash === -1) continue;
1056
+ mapInsert(dirGroups, file.slice(0, lastSlash + 1), () => []).push(file);
1057
+ }
1058
+ if (dirGroups.size === 0) return null;
1059
+ const scored = Array.from(dirGroups.entries(), ([dir, files]) => ({
1060
+ dir,
1061
+ files,
1062
+ score: scoreDocDir(dir, files.length)
1063
+ })).filter((d) => d.files.length >= 5).sort((a, b) => b.score - a.score);
1064
+ if (scored.length === 0) return null;
1065
+ const best = scored[0];
1066
+ return {
1067
+ files: best.files,
1068
+ prefix: best.dir
1069
+ };
1249
1070
  }
1250
- function readLocalSkill(dir, repoPath) {
1251
- const skillMdPath = resolve(dir, "SKILL.md");
1252
- if (!existsSync(skillMdPath)) return null;
1253
- const content = readFileSync(skillMdPath, "utf-8");
1254
- const frontmatter = parseSkillFrontmatterName(content);
1255
- const dirName = dir.split("/").pop();
1256
- const name = frontmatter.name || dirName;
1257
- const files = collectFiles(dir).filter((f) => f.path !== "SKILL.md");
1071
+ async function listDocsAtRef(owner, repo, ref, pathPrefix = "docs/") {
1072
+ return filterDocFiles(await listFilesAtRef(owner, repo, ref), pathPrefix);
1073
+ }
1074
+ async function fetchGitDocs(owner, repo, version, packageName, repoUrl) {
1075
+ const override = packageName ? getDocOverride(packageName) : void 0;
1076
+ if (override) {
1077
+ const ref = override.ref || "main";
1078
+ const fallback = !override.ref;
1079
+ const files = await listDocsAtRef(override.owner, override.repo, ref, `${override.path}/`);
1080
+ if (files.length === 0) return null;
1081
+ return {
1082
+ baseUrl: `https://raw.githubusercontent.com/${override.owner}/${override.repo}/${ref}`,
1083
+ ref,
1084
+ files,
1085
+ fallback,
1086
+ docsPrefix: `${override.path}/` !== "docs/" ? `${override.path}/` : void 0
1087
+ };
1088
+ }
1089
+ const tag = await findGitTag(owner, repo, version, packageName, repoUrl ? extractBranchHint(repoUrl) : void 0);
1090
+ if (!tag) return null;
1091
+ let docs = filterDocFiles(tag.files, "docs/");
1092
+ let docsPrefix;
1093
+ let allFiles;
1094
+ if (docs.length === 0) {
1095
+ const discovered = discoverDocFiles(tag.files, packageName);
1096
+ if (discovered) {
1097
+ docs = discovered.files;
1098
+ docsPrefix = discovered.prefix || void 0;
1099
+ allFiles = tag.files;
1100
+ }
1101
+ }
1102
+ docs = filterFrameworkDocs(docs, packageName);
1103
+ if (docs.length === 0) return null;
1258
1104
  return {
1259
- name,
1260
- description: frontmatter.description || "",
1261
- path: repoPath,
1262
- content,
1263
- files
1105
+ baseUrl: `https://raw.githubusercontent.com/${owner}/${repo}/${tag.ref}`,
1106
+ ref: tag.ref,
1107
+ files: docs,
1108
+ docsPrefix,
1109
+ allFiles,
1110
+ fallback: tag.fallback
1264
1111
  };
1265
1112
  }
1266
- async function fetchGitHubSkills(source, onProgress) {
1267
- const { owner, repo } = source;
1268
- if (!owner || !repo) return { skills: [] };
1269
- const ref = source.ref || "main";
1270
- const refs = ref === "main" ? ["main", "master"] : [ref];
1271
- for (const tryRef of refs) {
1272
- const skills = await downloadGitHubSkills(owner, repo, tryRef, source.skillPath, onProgress);
1273
- if (skills.length > 0) return { skills };
1113
+ function normalizePath(p) {
1114
+ return p.replace(/^\//, "").replace(/\.(?:md|mdx)$/, "");
1115
+ }
1116
+ function validateGitDocsWithLlms(llmsLinks, repoFiles) {
1117
+ if (llmsLinks.length === 0) return {
1118
+ isValid: true,
1119
+ matchRatio: 1
1120
+ };
1121
+ const sample = llmsLinks.slice(0, 10);
1122
+ const normalizedLinks = sample.map((link) => {
1123
+ let path = link.url;
1124
+ if (path.startsWith("http")) try {
1125
+ path = new URL(path).pathname;
1126
+ } catch {}
1127
+ return normalizePath(path);
1128
+ });
1129
+ const repoNormalized = new Set(repoFiles.map(normalizePath));
1130
+ let matches = 0;
1131
+ for (const linkPath of normalizedLinks) for (const repoPath of repoNormalized) if (repoPath === linkPath || repoPath.endsWith(`/${linkPath}`)) {
1132
+ matches++;
1133
+ break;
1274
1134
  }
1275
- return { skills: [] };
1135
+ const matchRatio = matches / sample.length;
1136
+ return {
1137
+ isValid: matchRatio >= .3,
1138
+ matchRatio
1139
+ };
1276
1140
  }
1277
- async function downloadGitHubSkills(owner, repo, ref, skillPath, onProgress) {
1278
- const tempDir = join(tmpdir(), `skilld-${Date.now()}`);
1279
- try {
1280
- if (skillPath) {
1281
- onProgress?.(`Downloading ${owner}/${repo}/${skillPath}@${ref}`);
1282
- const { dir } = await downloadTemplate(`github:${owner}/${repo}/${skillPath}#${ref}`, {
1283
- dir: tempDir,
1284
- force: true,
1285
- auth: getGitHubToken() || void 0
1286
- });
1287
- const skill = readLocalSkill(dir, skillPath);
1288
- return skill ? [skill] : [];
1289
- }
1290
- onProgress?.(`Downloading ${owner}/${repo}/skills@${ref}`);
1141
+ async function verifyNpmRepo(owner, repo, packageName) {
1142
+ const base = `https://raw.githubusercontent.com/${owner}/${repo}/HEAD`;
1143
+ const paths = [
1144
+ "package.json",
1145
+ `packages/${packageName.replace(/^@.*\//, "")}/package.json`,
1146
+ `packages/${packageName.replace(/^@/, "").replace("/", "-")}/package.json`
1147
+ ];
1148
+ for (const path of paths) {
1149
+ const text = await fetchGitHubRaw(`${base}/${path}`);
1150
+ if (!text) continue;
1291
1151
  try {
1292
- const { dir } = await downloadTemplate(`github:${owner}/${repo}/skills#${ref}`, {
1293
- dir: tempDir,
1294
- force: true,
1295
- auth: getGitHubToken() || void 0
1296
- });
1297
- const skills = [];
1298
- for (const entry of readdirSync(dir, { withFileTypes: true })) {
1299
- if (!entry.isDirectory()) continue;
1300
- const skill = readLocalSkill(resolve(dir, entry.name), `skills/${entry.name}`);
1301
- if (skill) skills.push(skill);
1302
- }
1303
- if (skills.length > 0) {
1304
- onProgress?.(`Found ${skills.length} skill(s)`);
1305
- return skills;
1306
- }
1152
+ if (JSON.parse(text).name === packageName) return true;
1307
1153
  } catch {}
1308
- const content = await fetchGitHubRaw(`https://raw.githubusercontent.com/${owner}/${repo}/${ref}/SKILL.md`);
1309
- if (content) {
1310
- const fm = parseSkillFrontmatterName(content);
1311
- onProgress?.("Found 1 skill");
1312
- return [{
1313
- name: fm.name || repo,
1314
- description: fm.description || "",
1315
- path: "",
1316
- content,
1317
- files: []
1318
- }];
1319
- }
1320
- return [];
1321
- } catch {
1322
- return [];
1323
- } finally {
1324
- rmSync(tempDir, {
1325
- recursive: true,
1326
- force: true
1327
- });
1328
1154
  }
1155
+ return false;
1329
1156
  }
1330
- async function fetchGitLabSkills(source, onProgress) {
1331
- const { owner, repo } = source;
1332
- if (!owner || !repo) return { skills: [] };
1333
- const ref = source.ref || "main";
1334
- const tempDir = join(tmpdir(), `skilld-gitlab-${Date.now()}`);
1335
- try {
1336
- const subdir = source.skillPath || "skills";
1337
- onProgress?.(`Downloading ${owner}/${repo}/${subdir}@${ref}`);
1338
- const { dir } = await downloadTemplate(`gitlab:${owner}/${repo}/${subdir}#${ref}`, {
1339
- dir: tempDir,
1340
- force: true
1341
- });
1342
- if (source.skillPath) {
1343
- const skill = readLocalSkill(dir, source.skillPath);
1344
- return { skills: skill ? [skill] : [] };
1345
- }
1346
- const skills = [];
1347
- for (const entry of readdirSync(dir, { withFileTypes: true })) {
1348
- if (!entry.isDirectory()) continue;
1349
- const skill = readLocalSkill(resolve(dir, entry.name), `skills/${entry.name}`);
1350
- if (skill) skills.push(skill);
1351
- }
1352
- if (skills.length > 0) {
1353
- onProgress?.(`Found ${skills.length} skill(s)`);
1354
- return { skills };
1355
- }
1356
- const content = await $fetch(`https://gitlab.com/${owner}/${repo}/-/raw/${ref}/SKILL.md`, { responseType: "text" }).catch(() => null);
1357
- if (content) {
1358
- const fm = parseSkillFrontmatterName(content);
1359
- return { skills: [{
1360
- name: fm.name || repo,
1361
- description: fm.description || "",
1362
- path: "",
1363
- content,
1364
- files: []
1365
- }] };
1157
+ async function searchGitHubRepo(packageName) {
1158
+ const shortName = packageName.replace(/^@.*\//, "");
1159
+ for (const candidate of [packageName.replace(/^@/, "").replace("/", "/"), shortName]) {
1160
+ if (!candidate.includes("/")) {
1161
+ if ((await $fetch.raw(`https://ungh.cc/repos/${shortName}/${shortName}`).catch(() => null))?.ok) return `https://github.com/${shortName}/${shortName}`;
1162
+ continue;
1366
1163
  }
1367
- return { skills: [] };
1368
- } catch {
1369
- return { skills: [] };
1370
- } finally {
1371
- rmSync(tempDir, {
1372
- recursive: true,
1373
- force: true
1164
+ if ((await $fetch.raw(`https://ungh.cc/repos/${candidate}`).catch(() => null))?.ok) return `https://github.com/${candidate}`;
1165
+ }
1166
+ const searchTerm = packageName.replace(/^@/, "");
1167
+ if (isGhAvailable()) try {
1168
+ const { stdout: json } = spawnSync("gh", [
1169
+ "search",
1170
+ "repos",
1171
+ searchTerm,
1172
+ "--json",
1173
+ "fullName",
1174
+ "--limit",
1175
+ "5"
1176
+ ], {
1177
+ encoding: "utf-8",
1178
+ timeout: 15e3
1374
1179
  });
1180
+ if (!json) throw new Error("no output");
1181
+ const repos = JSON.parse(json);
1182
+ const match = repos.find((r) => r.fullName.toLowerCase().endsWith(`/${packageName.toLowerCase()}`) || r.fullName.toLowerCase().endsWith(`/${shortName.toLowerCase()}`));
1183
+ if (match) return `https://github.com/${match.fullName}`;
1184
+ for (const candidate of repos) {
1185
+ const gh = parseGitHubUrl(`https://github.com/${candidate.fullName}`);
1186
+ if (gh && await verifyNpmRepo(gh.owner, gh.repo, packageName)) return `https://github.com/${candidate.fullName}`;
1187
+ }
1188
+ } catch {}
1189
+ const data = await $fetch(`https://api.github.com/search/repositories?q=${encodeURIComponent(`${searchTerm} in:name`)}&per_page=5`).catch(() => null);
1190
+ if (!data?.items?.length) return null;
1191
+ const match = data.items.find((r) => r.full_name.toLowerCase().endsWith(`/${packageName.toLowerCase()}`) || r.full_name.toLowerCase().endsWith(`/${shortName.toLowerCase()}`));
1192
+ if (match) return `https://github.com/${match.full_name}`;
1193
+ for (const candidate of data.items) {
1194
+ const gh = parseGitHubUrl(`https://github.com/${candidate.full_name}`);
1195
+ if (gh && await verifyNpmRepo(gh.owner, gh.repo, packageName)) return `https://github.com/${candidate.full_name}`;
1375
1196
  }
1376
- }
1377
- async function fetchLlmsUrl(docsUrl) {
1378
- const llmsUrl = `${new URL(docsUrl).origin}/llms.txt`;
1379
- if (await verifyUrl(llmsUrl)) return llmsUrl;
1380
1197
  return null;
1381
1198
  }
1382
- async function fetchLlmsTxt(url) {
1383
- const content = await fetchText(url);
1384
- if (!content || content.length < 50) return null;
1385
- return {
1386
- raw: content,
1387
- links: parseMarkdownLinks(content)
1388
- };
1389
- }
1390
- function parseMarkdownLinks(content) {
1391
- return extractLinks(content).filter((l) => l.url.endsWith(".md"));
1392
- }
1393
- function isSafeUrl(url) {
1394
- try {
1395
- const parsed = new URL(url);
1396
- if (parsed.protocol !== "https:") return false;
1397
- const host = parsed.hostname;
1398
- if (host === "localhost" || host === "0.0.0.0" || host === "[::1]") return false;
1399
- if (/^(?:127\.|10\.|172\.(?:1[6-9]|2\d|3[01])\.|192\.168\.|169\.254\.)/.test(host)) return false;
1400
- if (/^\[(?:f[cd]|fe[89ab]|::ffff:)/i.test(host)) return false;
1401
- return true;
1402
- } catch {
1403
- return false;
1404
- }
1405
- }
1406
- async function downloadLlmsDocs(llmsContent, baseUrl, onProgress) {
1407
- const limit = pLimit(5);
1408
- let completed = 0;
1409
- return (await Promise.all(llmsContent.links.map((link) => limit(async () => {
1410
- const url = link.url.startsWith("http") ? link.url : `${baseUrl.replace(/\/$/, "")}${link.url.startsWith("/") ? "" : "/"}${link.url}`;
1411
- if (!isSafeUrl(url)) return null;
1412
- const content = await fetchText(url);
1413
- onProgress?.(link.url, ++completed, llmsContent.links.length);
1414
- if (content && content.length > 100) return {
1415
- url: link.url.startsWith("http") ? new URL(link.url).pathname : link.url,
1416
- title: link.title,
1417
- content
1418
- };
1419
- return null;
1420
- })))).filter((d) => d !== null);
1421
- }
1422
- function normalizeLlmsLinks(content, baseUrl) {
1423
- let normalized = content;
1424
- if (baseUrl) {
1425
- const escaped = baseUrl.replace(/\/$/, "").replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
1426
- normalized = normalized.replace(new RegExp(`\\]\\(${escaped}(/[^)]+\\.md)\\)`, "g"), "](./docs$1)");
1427
- }
1428
- normalized = normalized.replace(/\]\(\/([^)]+\.md)\)/g, "](./docs/$1)");
1429
- return normalized;
1430
- }
1431
- function extractSections(content, patterns) {
1432
- const sections = [];
1433
- const parts = content.split(/\n---\n/);
1434
- for (const part of parts) {
1435
- const urlMatch = part.match(/^url: *(\S.*)$/m);
1436
- if (!urlMatch) continue;
1437
- const url = urlMatch[1];
1438
- if (patterns.some((p) => url.includes(p))) {
1439
- const contentStart = part.indexOf("\n", part.indexOf("url:"));
1440
- if (contentStart > -1) sections.push(part.slice(contentStart + 1));
1441
- }
1442
- }
1443
- if (sections.length === 0) return null;
1444
- return sections.join("\n\n---\n\n");
1199
+ async function fetchGitHubRepoMeta(owner, repo, packageName) {
1200
+ const override = packageName ? getDocOverride(packageName) : void 0;
1201
+ if (override?.homepage) return { homepage: override.homepage };
1202
+ const data = await ghApi(`repos/${owner}/${repo}`) ?? await $fetch(`https://api.github.com/repos/${owner}/${repo}`).catch(() => null);
1203
+ return data?.homepage ? { homepage: data.homepage } : null;
1445
1204
  }
1446
- const MIN_GIT_DOCS = 5;
1447
- const isShallowGitDocs = (n) => n > 0 && n < 5;
1448
- async function listFilesAtRef(owner, repo, ref) {
1205
+ async function fetchReadme(owner, repo, subdir, ref) {
1206
+ const branch = ref || "main";
1449
1207
  if (!isKnownPrivateRepo(owner, repo)) {
1450
- const data = await $fetch(`https://ungh.cc/repos/${owner}/${repo}/files/${ref}`).catch(() => null);
1451
- if (data?.files?.length) return data.files.map((f) => f.path);
1208
+ const unghUrl = subdir ? `https://ungh.cc/repos/${owner}/${repo}/files/${branch}/${subdir}/README.md` : `https://ungh.cc/repos/${owner}/${repo}/readme${ref ? `?ref=${ref}` : ""}`;
1209
+ if ((await $fetch.raw(unghUrl).catch(() => null))?.ok) return `ungh://${owner}/${repo}${subdir ? `/${subdir}` : ""}${ref ? `@${ref}` : ""}`;
1452
1210
  }
1453
- const tree = await ghApi(`repos/${owner}/${repo}/git/trees/${ref}?recursive=1`);
1454
- if (tree?.tree?.length) {
1211
+ const basePath = subdir ? `${subdir}/` : "";
1212
+ const branches = ref ? [ref] : ["main", "master"];
1213
+ const token = isKnownPrivateRepo(owner, repo) ? getGitHubToken() : null;
1214
+ const authHeaders = token ? { Authorization: `token ${token}` } : {};
1215
+ for (const b of branches) for (const filename of [
1216
+ "README.md",
1217
+ "Readme.md",
1218
+ "readme.md"
1219
+ ]) {
1220
+ const readmeUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${b}/${basePath}${filename}`;
1221
+ if ((await $fetch.raw(readmeUrl, { headers: authHeaders }).catch(() => null))?.ok) return readmeUrl;
1222
+ }
1223
+ const refParam = ref ? `?ref=${ref}` : "";
1224
+ const apiData = await ghApi(subdir ? `repos/${owner}/${repo}/contents/${subdir}/README.md${refParam}` : `repos/${owner}/${repo}/readme${refParam}`);
1225
+ if (apiData?.download_url) {
1455
1226
  markRepoPrivate(owner, repo);
1456
- return tree.tree.map((f) => f.path);
1227
+ return apiData.download_url;
1457
1228
  }
1458
- return [];
1229
+ return null;
1459
1230
  }
1460
- async function findGitTag(owner, repo, version, packageName, branchHint) {
1461
- const candidates = [`v${version}`, version];
1462
- if (packageName) candidates.push(`${packageName}@${version}`);
1463
- for (const tag of candidates) {
1464
- const files = await listFilesAtRef(owner, repo, tag);
1465
- if (files.length > 0) return {
1466
- ref: tag,
1467
- files
1468
- };
1231
+ async function fetchReadmeContent(url) {
1232
+ if (url.startsWith("file://")) {
1233
+ const filePath = fileURLToPath(url);
1234
+ if (!existsSync(filePath)) return null;
1235
+ return readFileSync(filePath, "utf-8");
1469
1236
  }
1470
- if (packageName) {
1471
- const latestTag = await findLatestReleaseTag(owner, repo, packageName);
1472
- if (latestTag) {
1473
- const files = await listFilesAtRef(owner, repo, latestTag);
1474
- if (files.length > 0) return {
1475
- ref: latestTag,
1476
- files
1477
- };
1237
+ if (url.startsWith("ungh://")) {
1238
+ let path = url.replace("ungh://", "");
1239
+ let ref = "main";
1240
+ const atIdx = path.lastIndexOf("@");
1241
+ if (atIdx !== -1) {
1242
+ ref = path.slice(atIdx + 1);
1243
+ path = path.slice(0, atIdx);
1244
+ }
1245
+ const parts = path.split("/");
1246
+ const owner = parts[0];
1247
+ const repo = parts[1];
1248
+ const subdir = parts.slice(2).join("/");
1249
+ const text = await $fetch(subdir ? `https://ungh.cc/repos/${owner}/${repo}/files/${ref}/${subdir}/README.md` : `https://ungh.cc/repos/${owner}/${repo}/readme?ref=${ref}`, { responseType: "text" }).catch(() => null);
1250
+ if (!text) return null;
1251
+ try {
1252
+ const json = JSON.parse(text);
1253
+ return json.markdown || json.file?.contents || null;
1254
+ } catch {
1255
+ return text;
1478
1256
  }
1479
1257
  }
1480
- const branches = branchHint ? [branchHint, ...["main", "master"].filter((b) => b !== branchHint)] : ["main", "master"];
1481
- for (const branch of branches) {
1482
- const files = await listFilesAtRef(owner, repo, branch);
1483
- if (files.length > 0) return {
1484
- ref: branch,
1485
- files,
1486
- fallback: true
1258
+ if (url.includes("raw.githubusercontent.com")) return fetchGitHubRaw(url);
1259
+ return fetchText(url);
1260
+ }
1261
+ async function resolveGitHubRepo(owner, repo, onProgress) {
1262
+ onProgress?.("Fetching repo metadata");
1263
+ const repoUrl = `https://github.com/${owner}/${repo}`;
1264
+ const meta = await ghApi(`repos/${owner}/${repo}`) ?? await $fetch(`https://api.github.com/repos/${owner}/${repo}`).catch(() => null);
1265
+ const homepage = meta?.homepage || void 0;
1266
+ const description = meta?.description || void 0;
1267
+ onProgress?.("Fetching latest release");
1268
+ const releases = await fetchUnghReleases(owner, repo);
1269
+ let version = "main";
1270
+ let releasedAt;
1271
+ const latestRelease = releases[0];
1272
+ if (latestRelease) {
1273
+ version = latestRelease.tag.replace(/^v/, "");
1274
+ releasedAt = latestRelease.publishedAt;
1275
+ }
1276
+ onProgress?.("Resolving docs");
1277
+ const gitDocs = await fetchGitDocs(owner, repo, version);
1278
+ const gitDocsUrl = gitDocs ? `${repoUrl}/tree/${gitDocs.ref}/docs` : void 0;
1279
+ const gitRef = gitDocs?.ref;
1280
+ onProgress?.("Fetching README");
1281
+ const readmeUrl = await fetchReadme(owner, repo);
1282
+ let llmsUrl;
1283
+ if (homepage) {
1284
+ onProgress?.("Checking llms.txt");
1285
+ llmsUrl = await fetchLlmsUrl(homepage).catch(() => null) ?? void 0;
1286
+ }
1287
+ if (!gitDocsUrl && !readmeUrl && !llmsUrl) return null;
1288
+ return {
1289
+ name: repo,
1290
+ version: latestRelease ? version : void 0,
1291
+ releasedAt,
1292
+ description,
1293
+ repoUrl,
1294
+ docsUrl: homepage,
1295
+ gitDocsUrl,
1296
+ gitRef,
1297
+ gitDocsFallback: gitDocs?.fallback,
1298
+ readmeUrl: readmeUrl ?? void 0,
1299
+ llmsUrl
1300
+ };
1301
+ }
1302
+ const VALID_CRATE_NAME = /^[a-z0-9][\w-]*$/;
1303
+ const runCratesApiRateLimited = createRateLimitedRunner(1e3);
1304
+ function selectCrateVersion(data, requestedVersion) {
1305
+ const versions = data.versions || [];
1306
+ if (requestedVersion) {
1307
+ const exact = versions.find((v) => v.num === requestedVersion && !v.yanked);
1308
+ if (exact?.num) return {
1309
+ version: exact.num,
1310
+ entry: exact
1487
1311
  };
1488
1312
  }
1313
+ const crate = data.crate;
1314
+ const preferred = [
1315
+ crate?.max_stable_version,
1316
+ crate?.newest_version,
1317
+ crate?.max_version,
1318
+ crate?.default_version
1319
+ ].find(Boolean);
1320
+ if (preferred) {
1321
+ const match = versions.find((v) => v.num === preferred && !v.yanked);
1322
+ if (match?.num) return {
1323
+ version: preferred,
1324
+ entry: match
1325
+ };
1326
+ if (versions.length === 0) return { version: preferred };
1327
+ }
1328
+ const firstStable = versions.find((v) => !v.yanked && v.num);
1329
+ if (firstStable?.num) return {
1330
+ version: firstStable.num,
1331
+ entry: firstStable
1332
+ };
1489
1333
  return null;
1490
1334
  }
1491
- async function fetchUnghReleases(owner, repo) {
1492
- if (!isKnownPrivateRepo(owner, repo)) {
1493
- const data = await $fetch(`https://ungh.cc/repos/${owner}/${repo}/releases`).catch(() => null);
1494
- if (data?.releases?.length) return data.releases;
1335
+ function pickPreferredUrl(...urls) {
1336
+ return urls.map((v) => v?.trim()).find((v) => !!v);
1337
+ }
1338
+ async function fetchCratesApi(url) {
1339
+ return runCratesApiRateLimited(() => $fetch(url).catch(() => null));
1340
+ }
1341
+ async function resolveCrateDocsWithAttempts(crateName, options = {}) {
1342
+ const attempts = [];
1343
+ const onProgress = options.onProgress;
1344
+ const normalizedName = crateName.trim().toLowerCase();
1345
+ if (!normalizedName || !VALID_CRATE_NAME.test(normalizedName)) {
1346
+ attempts.push({
1347
+ source: "crates",
1348
+ status: "error",
1349
+ message: `Invalid crate name: ${crateName}`
1350
+ });
1351
+ return {
1352
+ package: null,
1353
+ attempts
1354
+ };
1495
1355
  }
1496
- const raw = await ghApiPaginated(`repos/${owner}/${repo}/releases`);
1497
- if (raw.length > 0) {
1498
- markRepoPrivate(owner, repo);
1499
- return raw.map((r) => ({
1500
- tag: r.tag_name,
1501
- publishedAt: r.published_at
1502
- }));
1356
+ onProgress?.("crates.io metadata");
1357
+ const apiUrl = `https://crates.io/api/v1/crates/${encodeURIComponent(normalizedName)}`;
1358
+ const data = await fetchCratesApi(apiUrl);
1359
+ if (!data?.crate) {
1360
+ attempts.push({
1361
+ source: "crates",
1362
+ url: apiUrl,
1363
+ status: "not-found",
1364
+ message: "Crate not found on crates.io"
1365
+ });
1366
+ return {
1367
+ package: null,
1368
+ attempts
1369
+ };
1503
1370
  }
1504
- return [];
1505
- }
1506
- async function findLatestReleaseTag(owner, repo, packageName) {
1507
- const prefix = `${packageName}@`;
1508
- return (await fetchUnghReleases(owner, repo)).find((r) => r.tag.startsWith(prefix))?.tag ?? null;
1371
+ attempts.push({
1372
+ source: "crates",
1373
+ url: apiUrl,
1374
+ status: "success",
1375
+ message: `Found crate: ${data.crate.name || normalizedName}`
1376
+ });
1377
+ const selected = selectCrateVersion(data, options.version);
1378
+ if (!selected) {
1379
+ attempts.push({
1380
+ source: "crates",
1381
+ url: apiUrl,
1382
+ status: "error",
1383
+ message: "No usable crate versions found"
1384
+ });
1385
+ return {
1386
+ package: null,
1387
+ attempts
1388
+ };
1389
+ }
1390
+ const version = selected.version;
1391
+ const versionEntry = selected.entry;
1392
+ const docsRsUrl = `https://docs.rs/${encodeURIComponent(normalizedName)}/${encodeURIComponent(version)}`;
1393
+ const repositoryRaw = pickPreferredUrl(versionEntry?.repository, data.crate.repository);
1394
+ const homepage = pickPreferredUrl(versionEntry?.homepage, data.crate.homepage);
1395
+ const documentation = pickPreferredUrl(versionEntry?.documentation, data.crate.documentation);
1396
+ const normalizedRepo = repositoryRaw ? normalizeRepoUrl(repositoryRaw) : void 0;
1397
+ const repoUrl = normalizedRepo && isLikelyCodeHostUrl(normalizedRepo) ? normalizedRepo : isLikelyCodeHostUrl(homepage) ? homepage : void 0;
1398
+ let resolved = {
1399
+ name: normalizedName,
1400
+ version,
1401
+ releasedAt: versionEntry?.created_at || data.crate.updated_at || void 0,
1402
+ description: versionEntry?.description || data.crate.description,
1403
+ docsUrl: (() => {
1404
+ if (documentation && !isUselessDocsUrl(documentation) && !isLikelyCodeHostUrl(documentation)) return documentation;
1405
+ if (homepage && !isUselessDocsUrl(homepage) && !isLikelyCodeHostUrl(homepage)) return homepage;
1406
+ return docsRsUrl;
1407
+ })(),
1408
+ repoUrl
1409
+ };
1410
+ const gh = repoUrl ? parseGitHubUrl(repoUrl) : null;
1411
+ if (gh) {
1412
+ onProgress?.("GitHub enrichment");
1413
+ const ghResolved = await resolveGitHubRepo(gh.owner, gh.repo);
1414
+ if (ghResolved) {
1415
+ attempts.push({
1416
+ source: "github-meta",
1417
+ url: repoUrl,
1418
+ status: "success",
1419
+ message: "Enriched via GitHub repo metadata"
1420
+ });
1421
+ resolved = {
1422
+ ...ghResolved,
1423
+ name: normalizedName,
1424
+ version,
1425
+ releasedAt: resolved.releasedAt || ghResolved.releasedAt,
1426
+ description: resolved.description || ghResolved.description,
1427
+ docsUrl: resolved.docsUrl || ghResolved.docsUrl,
1428
+ repoUrl,
1429
+ readmeUrl: ghResolved.readmeUrl || resolved.readmeUrl
1430
+ };
1431
+ } else attempts.push({
1432
+ source: "github-meta",
1433
+ url: repoUrl,
1434
+ status: "not-found",
1435
+ message: "GitHub enrichment failed, using crates.io metadata"
1436
+ });
1437
+ }
1438
+ if (!resolved.llmsUrl && resolved.docsUrl) {
1439
+ onProgress?.("llms.txt discovery");
1440
+ resolved.llmsUrl = await fetchLlmsUrl(resolved.docsUrl).catch(() => null) ?? void 0;
1441
+ if (resolved.llmsUrl) attempts.push({
1442
+ source: "llms.txt",
1443
+ url: resolved.llmsUrl,
1444
+ status: "success"
1445
+ });
1446
+ }
1447
+ return {
1448
+ package: resolved,
1449
+ attempts
1450
+ };
1509
1451
  }
1510
- function filterDocFiles(files, pathPrefix) {
1511
- return files.filter((f) => f.startsWith(pathPrefix) && /\.(?:md|mdx)$/.test(f));
1452
+ async function fetchCrawledDocs(url, onProgress, maxPages = 200) {
1453
+ const outputDir = join(tmpdir(), "skilld-crawl", Date.now().toString());
1454
+ onProgress?.(`Crawling ${url}`);
1455
+ const userLang = getUserLang();
1456
+ const foreignUrls = /* @__PURE__ */ new Set();
1457
+ const doCrawl = () => crawlAndGenerate({
1458
+ urls: [url],
1459
+ outputDir,
1460
+ driver: "http",
1461
+ generateLlmsTxt: false,
1462
+ generateIndividualMd: true,
1463
+ maxRequestsPerCrawl: maxPages,
1464
+ onPage: (page) => {
1465
+ const lang = extractHtmlLang(page.html);
1466
+ if (lang && !lang.startsWith("en") && !lang.startsWith(userLang)) foreignUrls.add(page.url);
1467
+ }
1468
+ }, (progress) => {
1469
+ if (progress.crawling.status === "processing" && progress.crawling.total > 0) onProgress?.(`Crawling ${progress.crawling.processed}/${progress.crawling.total} pages`);
1470
+ });
1471
+ let results = await doCrawl().catch((err) => {
1472
+ onProgress?.(`Crawl failed: ${err?.message || err}`);
1473
+ return [];
1474
+ });
1475
+ if (results.length === 0) {
1476
+ onProgress?.("Retrying crawl");
1477
+ results = await doCrawl().catch(() => []);
1478
+ }
1479
+ rmSync(outputDir, {
1480
+ recursive: true,
1481
+ force: true
1482
+ });
1483
+ const docs = [];
1484
+ let localeFiltered = 0;
1485
+ for (const result of results) {
1486
+ if (!result.success || !result.content) continue;
1487
+ if (foreignUrls.has(result.url)) {
1488
+ localeFiltered++;
1489
+ continue;
1490
+ }
1491
+ const segments = (new URL(result.url).pathname.replace(/\/$/, "") || "/index").split("/").filter(Boolean);
1492
+ if (isForeignPathPrefix(segments[0], userLang)) {
1493
+ localeFiltered++;
1494
+ continue;
1495
+ }
1496
+ const path = `docs/${segments.join("/")}.md`;
1497
+ docs.push({
1498
+ path,
1499
+ content: result.content
1500
+ });
1501
+ }
1502
+ if (localeFiltered > 0) onProgress?.(`Filtered ${localeFiltered} foreign locale pages`);
1503
+ onProgress?.(`Crawled ${docs.length} pages`);
1504
+ return docs;
1512
1505
  }
1513
- const FRAMEWORK_NAMES = new Set([
1514
- "vue",
1515
- "react",
1516
- "solid",
1517
- "angular",
1518
- "svelte",
1519
- "preact",
1520
- "lit",
1521
- "qwik"
1522
- ]);
1523
- function filterFrameworkDocs(files, packageName) {
1524
- if (!packageName) return files;
1525
- const shortName = packageName.replace(/^@.*\//, "");
1526
- const targetFramework = [...FRAMEWORK_NAMES].find((fw) => shortName.includes(fw));
1527
- if (!targetFramework) return files;
1528
- const otherFrameworks = [...FRAMEWORK_NAMES].filter((fw) => fw !== targetFramework);
1529
- const excludePattern = new RegExp(`\\b(?:${otherFrameworks.join("|")})\\b`);
1530
- return files.filter((f) => !excludePattern.test(f));
1506
+ const HTML_LANG_RE = /<html[^>]*\slang=["']([^"']+)["']/i;
1507
+ function extractHtmlLang(html) {
1508
+ return HTML_LANG_RE.exec(html)?.[1]?.toLowerCase();
1531
1509
  }
1532
- const NOISE_PATTERNS = [
1533
- /^\.changeset\//,
1534
- /CHANGELOG\.md$/i,
1535
- /CONTRIBUTING\.md$/i,
1536
- /^\.github\//
1537
- ];
1538
- const EXCLUDE_DIRS = new Set([
1539
- "test",
1540
- "tests",
1541
- "__tests__",
1542
- "fixtures",
1543
- "fixture",
1544
- "examples",
1545
- "example",
1546
- "node_modules",
1547
- ".git",
1548
- "dist",
1549
- "build",
1550
- "coverage",
1551
- "e2e",
1552
- "spec",
1553
- "mocks",
1554
- "__mocks__"
1555
- ]);
1556
- const DOC_DIR_BONUS = new Set([
1557
- "docs",
1558
- "documentation",
1559
- "pages",
1560
- "content",
1561
- "website",
1562
- "guide",
1563
- "guides",
1564
- "wiki",
1565
- "manual",
1566
- "api"
1510
+ const LOCALE_CODES = new Set([
1511
+ "ar",
1512
+ "de",
1513
+ "es",
1514
+ "fr",
1515
+ "id",
1516
+ "it",
1517
+ "ja",
1518
+ "ko",
1519
+ "nl",
1520
+ "pl",
1521
+ "pt",
1522
+ "pt-br",
1523
+ "ru",
1524
+ "th",
1525
+ "tr",
1526
+ "uk",
1527
+ "vi",
1528
+ "zh",
1529
+ "zh-cn",
1530
+ "zh-tw"
1567
1531
  ]);
1568
- function hasExcludedDir(path) {
1569
- return path.split("/").some((p) => EXCLUDE_DIRS.has(p.toLowerCase()));
1532
+ function isForeignPathPrefix(segment, userLang) {
1533
+ if (!segment) return false;
1534
+ const lower = segment.toLowerCase();
1535
+ if (lower === "en" || lower.startsWith(userLang)) return false;
1536
+ return LOCALE_CODES.has(lower);
1570
1537
  }
1571
- function getPathDepth(path) {
1572
- return path.split("/").filter(Boolean).length;
1538
+ function getUserLang() {
1539
+ const code = (process.env.LC_ALL || process.env.LANG || process.env.LANGUAGE || "").split(/[_.:-]/)[0]?.toLowerCase() || "";
1540
+ return code.length >= 2 ? code.slice(0, 2) : "en";
1573
1541
  }
1574
- function hasDocDirBonus(path) {
1575
- return path.split("/").some((p) => DOC_DIR_BONUS.has(p.toLowerCase()));
1542
+ function toCrawlPattern(docsUrl) {
1543
+ return `${docsUrl.replace(/\/+$/, "")}/**`;
1576
1544
  }
1577
- function scoreDocDir(dir, fileCount) {
1578
- const depth = getPathDepth(dir) || 1;
1579
- return fileCount * (hasDocDirBonus(dir) ? 1.5 : 1) / depth;
1545
+ const HIGH_VALUE_CATEGORIES = new Set([
1546
+ "q&a",
1547
+ "help",
1548
+ "troubleshooting",
1549
+ "support"
1550
+ ]);
1551
+ const LOW_VALUE_CATEGORIES = new Set([
1552
+ "show and tell",
1553
+ "ideas",
1554
+ "polls"
1555
+ ]);
1556
+ const TITLE_NOISE_RE = /looking .*(?:developer|engineer|freelanc)|hiring|job post|guide me to (?:complete|finish|build)|help me (?:complete|finish|build)|seeking .* tutorial|recommend.* course/i;
1557
+ const MIN_DISCUSSION_SCORE = 3;
1558
+ function scoreComment(c) {
1559
+ return (c.isMaintainer ? 3 : 1) * (hasCodeBlock(c.body) ? 2 : 1) * (1 + c.reactions);
1580
1560
  }
1581
- function discoverDocFiles(allFiles, packageName) {
1582
- const mdFiles = allFiles.filter((f) => /\.(?:md|mdx)$/.test(f)).filter((f) => !NOISE_PATTERNS.some((p) => p.test(f))).filter((f) => f.includes("/"));
1583
- if (packageName?.includes("/")) {
1584
- const subPkgPrefix = `packages/${packageName.split("/").pop().toLowerCase()}/`;
1585
- const subPkgFiles = mdFiles.filter((f) => f.startsWith(subPkgPrefix));
1586
- if (subPkgFiles.length >= 3) return {
1587
- files: subPkgFiles,
1588
- prefix: subPkgPrefix
1589
- };
1561
+ function scoreDiscussion(d) {
1562
+ if (TITLE_NOISE_RE.test(d.title)) return -1;
1563
+ let score = 0;
1564
+ if (d.isMaintainer) score += 3;
1565
+ if (hasCodeBlock([
1566
+ d.body,
1567
+ d.answer || "",
1568
+ ...d.topComments.map((c) => c.body)
1569
+ ].join("\n"))) score += 3;
1570
+ score += Math.min(d.upvoteCount, 5);
1571
+ if (d.answer) {
1572
+ score += 2;
1573
+ if (d.answer.length > 100) score += 1;
1590
1574
  }
1591
- const docsGroups = /* @__PURE__ */ new Map();
1592
- for (const file of mdFiles) {
1593
- const docsIdx = file.lastIndexOf("/docs/");
1594
- if (docsIdx === -1) continue;
1595
- mapInsert(docsGroups, file.slice(0, docsIdx + 6), () => []).push(file);
1575
+ if (d.topComments.some((c) => c.isMaintainer)) score += 2;
1576
+ if (d.topComments.some((c) => c.reactions > 0)) score += 1;
1577
+ return score;
1578
+ }
1579
+ async function fetchGitHubDiscussions(owner, repo, limit = 20, releasedAt, fromDate) {
1580
+ if (!isGhAvailable()) return [];
1581
+ if (!fromDate && releasedAt) {
1582
+ const cutoff = new Date(releasedAt);
1583
+ cutoff.setMonth(cutoff.getMonth() + 6);
1584
+ if (cutoff < /* @__PURE__ */ new Date()) return [];
1596
1585
  }
1597
- if (docsGroups.size > 0) {
1598
- const largest = [...docsGroups.entries()].sort((a, b) => b[1].length - a[1].length)[0];
1599
- if (largest[1].length >= 3) {
1600
- const fullPrefix = largest[0];
1601
- const docsIdx = fullPrefix.lastIndexOf("docs/");
1602
- const stripPrefix = docsIdx > 0 ? fullPrefix.slice(0, docsIdx) : "";
1586
+ try {
1587
+ const { stdout: result } = spawnSync("gh", [
1588
+ "api",
1589
+ "graphql",
1590
+ "-f",
1591
+ `query=${`query($owner: String!, $repo: String!) { repository(owner: $owner, name: $repo) { discussions(first: ${Math.min(limit * 3, 80)}, orderBy: {field: CREATED_AT, direction: DESC}) { nodes { number title body category { name } createdAt url upvoteCount comments(first: 10) { totalCount nodes { body author { login } authorAssociation reactions { totalCount } } } answer { body author { login } authorAssociation } author { login } authorAssociation } } } }`}`,
1592
+ "-f",
1593
+ `owner=${owner}`,
1594
+ "-f",
1595
+ `repo=${repo}`
1596
+ ], {
1597
+ encoding: "utf-8",
1598
+ maxBuffer: 10 * 1024 * 1024
1599
+ });
1600
+ if (!result) return [];
1601
+ const nodes = JSON.parse(result)?.data?.repository?.discussions?.nodes;
1602
+ if (!Array.isArray(nodes)) return [];
1603
+ const fromTs = fromDate ? new Date(fromDate).getTime() : null;
1604
+ return nodes.filter((d) => d.author && !BOT_USERS.has(d.author.login)).filter((d) => {
1605
+ const cat = (d.category?.name || "").toLowerCase();
1606
+ return !LOW_VALUE_CATEGORIES.has(cat);
1607
+ }).filter((d) => !fromTs || new Date(d.createdAt).getTime() >= fromTs).map((d) => {
1608
+ let answer;
1609
+ if (d.answer?.body) {
1610
+ const isMaintainer = [
1611
+ "OWNER",
1612
+ "MEMBER",
1613
+ "COLLABORATOR"
1614
+ ].includes(d.answer.authorAssociation);
1615
+ const author = d.answer.author?.login;
1616
+ answer = `${isMaintainer && author ? `**@${author}** [maintainer]:\n\n` : ""}${d.answer.body}`;
1617
+ }
1618
+ const comments = (d.comments?.nodes || []).filter((c) => c.author && !BOT_USERS.has(c.author.login)).filter((c) => !COMMENT_NOISE_RE.test((c.body || "").trim())).map((c) => {
1619
+ const isMaintainer = [
1620
+ "OWNER",
1621
+ "MEMBER",
1622
+ "COLLABORATOR"
1623
+ ].includes(c.authorAssociation);
1624
+ return {
1625
+ body: c.body || "",
1626
+ author: c.author.login,
1627
+ reactions: c.reactions?.totalCount || 0,
1628
+ isMaintainer
1629
+ };
1630
+ }).sort((a, b) => scoreComment(b) - scoreComment(a)).slice(0, 3);
1603
1631
  return {
1604
- files: largest[1],
1605
- prefix: stripPrefix
1632
+ number: d.number,
1633
+ title: d.title,
1634
+ body: d.body || "",
1635
+ category: d.category?.name || "",
1636
+ createdAt: d.createdAt,
1637
+ url: d.url,
1638
+ upvoteCount: d.upvoteCount || 0,
1639
+ comments: d.comments?.totalCount || 0,
1640
+ isMaintainer: [
1641
+ "OWNER",
1642
+ "MEMBER",
1643
+ "COLLABORATOR"
1644
+ ].includes(d.authorAssociation),
1645
+ answer,
1646
+ topComments: comments
1606
1647
  };
1607
- }
1608
- }
1609
- const dirGroups = /* @__PURE__ */ new Map();
1610
- for (const file of mdFiles) {
1611
- if (hasExcludedDir(file)) continue;
1612
- const lastSlash = file.lastIndexOf("/");
1613
- if (lastSlash === -1) continue;
1614
- mapInsert(dirGroups, file.slice(0, lastSlash + 1), () => []).push(file);
1648
+ }).map((d) => ({
1649
+ d,
1650
+ score: scoreDiscussion(d)
1651
+ })).filter(({ score }) => score >= MIN_DISCUSSION_SCORE).sort((a, b) => {
1652
+ const aHigh = HIGH_VALUE_CATEGORIES.has(a.d.category.toLowerCase()) ? 1 : 0;
1653
+ const bHigh = HIGH_VALUE_CATEGORIES.has(b.d.category.toLowerCase()) ? 1 : 0;
1654
+ if (aHigh !== bHigh) return bHigh - aHigh;
1655
+ return b.score - a.score;
1656
+ }).slice(0, limit).map(({ d }) => d);
1657
+ } catch {
1658
+ return [];
1615
1659
  }
1616
- if (dirGroups.size === 0) return null;
1617
- const scored = Array.from(dirGroups.entries(), ([dir, files]) => ({
1618
- dir,
1619
- files,
1620
- score: scoreDocDir(dir, files.length)
1621
- })).filter((d) => d.files.length >= 5).sort((a, b) => b.score - a.score);
1622
- if (scored.length === 0) return null;
1623
- const best = scored[0];
1624
- return {
1625
- files: best.files,
1626
- prefix: best.dir
1627
- };
1628
- }
1629
- async function listDocsAtRef(owner, repo, ref, pathPrefix = "docs/") {
1630
- return filterDocFiles(await listFilesAtRef(owner, repo, ref), pathPrefix);
1631
1660
  }
1632
- async function fetchGitDocs(owner, repo, version, packageName, repoUrl) {
1633
- const override = packageName ? getDocOverride(packageName) : void 0;
1634
- if (override) {
1635
- const ref = override.ref || "main";
1636
- const fallback = !override.ref;
1637
- const files = await listDocsAtRef(override.owner, override.repo, ref, `${override.path}/`);
1638
- if (files.length === 0) return null;
1639
- return {
1640
- baseUrl: `https://raw.githubusercontent.com/${override.owner}/${override.repo}/${ref}`,
1641
- ref,
1642
- files,
1643
- fallback,
1644
- docsPrefix: `${override.path}/` !== "docs/" ? `${override.path}/` : void 0
1645
- };
1646
- }
1647
- const tag = await findGitTag(owner, repo, version, packageName, repoUrl ? extractBranchHint(repoUrl) : void 0);
1648
- if (!tag) return null;
1649
- let docs = filterDocFiles(tag.files, "docs/");
1650
- let docsPrefix;
1651
- let allFiles;
1652
- if (docs.length === 0) {
1653
- const discovered = discoverDocFiles(tag.files, packageName);
1654
- if (discovered) {
1655
- docs = discovered.files;
1656
- docsPrefix = discovered.prefix || void 0;
1657
- allFiles = tag.files;
1661
+ function formatDiscussionAsMarkdown(d) {
1662
+ const fm = buildFrontmatter({
1663
+ number: d.number,
1664
+ title: d.title,
1665
+ category: d.category,
1666
+ created: isoDate(d.createdAt),
1667
+ url: d.url,
1668
+ upvotes: d.upvoteCount,
1669
+ comments: d.comments,
1670
+ answered: !!d.answer
1671
+ });
1672
+ const bodyLimit = d.upvoteCount >= 5 ? 1500 : 800;
1673
+ const lines = [
1674
+ fm,
1675
+ "",
1676
+ `# ${d.title}`
1677
+ ];
1678
+ if (d.body) lines.push("", truncateBody(d.body, bodyLimit));
1679
+ if (d.answer) lines.push("", "---", "", "## Accepted Answer", "", truncateBody(d.answer, 1e3));
1680
+ else if (d.topComments.length > 0) {
1681
+ lines.push("", "---", "", "## Top Comments");
1682
+ for (const c of d.topComments) {
1683
+ const reactions = c.reactions > 0 ? ` (+${c.reactions})` : "";
1684
+ const maintainer = c.isMaintainer ? " [maintainer]" : "";
1685
+ lines.push("", `**@${c.author}**${maintainer}${reactions}:`, "", truncateBody(c.body, 600));
1658
1686
  }
1659
1687
  }
1660
- docs = filterFrameworkDocs(docs, packageName);
1661
- if (docs.length === 0) return null;
1662
- return {
1663
- baseUrl: `https://raw.githubusercontent.com/${owner}/${repo}/${tag.ref}`,
1664
- ref: tag.ref,
1665
- files: docs,
1666
- docsPrefix,
1667
- allFiles,
1668
- fallback: tag.fallback
1669
- };
1670
- }
1671
- function normalizePath(p) {
1672
- return p.replace(/^\//, "").replace(/\.(?:md|mdx)$/, "");
1688
+ return lines.join("\n");
1673
1689
  }
1674
- function validateGitDocsWithLlms(llmsLinks, repoFiles) {
1675
- if (llmsLinks.length === 0) return {
1676
- isValid: true,
1677
- matchRatio: 1
1678
- };
1679
- const sample = llmsLinks.slice(0, 10);
1680
- const normalizedLinks = sample.map((link) => {
1681
- let path = link.url;
1682
- if (path.startsWith("http")) try {
1683
- path = new URL(path).pathname;
1684
- } catch {}
1685
- return normalizePath(path);
1690
+ function generateDiscussionIndex(discussions) {
1691
+ const byCategory = /* @__PURE__ */ new Map();
1692
+ for (const d of discussions) mapInsert(byCategory, d.category || "Uncategorized", () => []).push(d);
1693
+ const answered = discussions.filter((d) => d.answer).length;
1694
+ const sections = [
1695
+ [
1696
+ "---",
1697
+ `total: ${discussions.length}`,
1698
+ `answered: ${answered}`,
1699
+ "---"
1700
+ ].join("\n"),
1701
+ "",
1702
+ "# Discussions Index",
1703
+ ""
1704
+ ];
1705
+ const cats = [...byCategory.keys()].sort((a, b) => {
1706
+ return (HIGH_VALUE_CATEGORIES.has(a.toLowerCase()) ? 0 : 1) - (HIGH_VALUE_CATEGORIES.has(b.toLowerCase()) ? 0 : 1) || a.localeCompare(b);
1686
1707
  });
1687
- const repoNormalized = new Set(repoFiles.map(normalizePath));
1688
- let matches = 0;
1689
- for (const linkPath of normalizedLinks) for (const repoPath of repoNormalized) if (repoPath === linkPath || repoPath.endsWith(`/${linkPath}`)) {
1690
- matches++;
1691
- break;
1708
+ for (const cat of cats) {
1709
+ const group = byCategory.get(cat);
1710
+ sections.push(`## ${cat} (${group.length})`, "");
1711
+ for (const d of group) {
1712
+ const upvotes = d.upvoteCount > 0 ? ` (+${d.upvoteCount})` : "";
1713
+ const answered = d.answer ? " [answered]" : "";
1714
+ const date = isoDate(d.createdAt);
1715
+ sections.push(`- [#${d.number}](./discussion-${d.number}.md): ${d.title}${upvotes}${answered} (${date})`);
1716
+ }
1717
+ sections.push("");
1692
1718
  }
1693
- const matchRatio = matches / sample.length;
1694
- return {
1695
- isValid: matchRatio >= .3,
1696
- matchRatio
1697
- };
1719
+ return sections.join("\n");
1698
1720
  }
1699
- async function verifyNpmRepo(owner, repo, packageName) {
1700
- const base = `https://raw.githubusercontent.com/${owner}/${repo}/HEAD`;
1701
- const paths = [
1702
- "package.json",
1703
- `packages/${packageName.replace(/^@.*\//, "")}/package.json`,
1704
- `packages/${packageName.replace(/^@/, "").replace("/", "-")}/package.json`
1721
+ function generateDocsIndex(docs) {
1722
+ const docFiles = docs.filter((d) => d.path.startsWith("docs/") && d.path.endsWith(".md") && !d.path.endsWith("_INDEX.md")).sort((a, b) => a.path.localeCompare(b.path));
1723
+ if (docFiles.length === 0) return "";
1724
+ const rootFiles = [];
1725
+ const byDir = /* @__PURE__ */ new Map();
1726
+ for (const doc of docFiles) {
1727
+ const rel = doc.path.slice(5);
1728
+ const dir = rel.includes("/") ? rel.slice(0, rel.lastIndexOf("/")) : "";
1729
+ if (!dir) rootFiles.push(doc);
1730
+ else {
1731
+ const list = byDir.get(dir);
1732
+ if (list) list.push(doc);
1733
+ else byDir.set(dir, [doc]);
1734
+ }
1735
+ }
1736
+ const sections = [
1737
+ "---",
1738
+ `total: ${docFiles.length}`,
1739
+ "---",
1740
+ "",
1741
+ "# Docs Index",
1742
+ ""
1705
1743
  ];
1706
- for (const path of paths) {
1707
- const text = await fetchGitHubRaw(`${base}/${path}`);
1708
- if (!text) continue;
1709
- try {
1710
- if (JSON.parse(text).name === packageName) return true;
1711
- } catch {}
1744
+ for (const file of rootFiles) {
1745
+ const rel = file.path.slice(5);
1746
+ const title = extractTitle(file.content) || rel.replace(/\.md$/, "");
1747
+ const desc = extractDescription(file.content);
1748
+ const descPart = desc ? `: ${desc}` : "";
1749
+ sections.push(`- [${title}](./${rel})${descPart}`);
1712
1750
  }
1713
- return false;
1751
+ if (rootFiles.length > 0) sections.push("");
1752
+ for (const [dir, files] of byDir) {
1753
+ sections.push(`## ${dir} (${files.length})`, "");
1754
+ for (const file of files) {
1755
+ const rel = file.path.slice(5);
1756
+ const title = extractTitle(file.content) || rel.replace(/\.md$/, "").split("/").pop();
1757
+ const desc = extractDescription(file.content);
1758
+ const descPart = desc ? `: ${desc}` : "";
1759
+ sections.push(`- [${title}](./${rel})${descPart}`);
1760
+ }
1761
+ sections.push("");
1762
+ }
1763
+ return sections.join("\n");
1714
1764
  }
1715
- async function searchGitHubRepo(packageName) {
1716
- const shortName = packageName.replace(/^@.*\//, "");
1717
- for (const candidate of [packageName.replace(/^@/, "").replace("/", "/"), shortName]) {
1718
- if (!candidate.includes("/")) {
1719
- if ((await $fetch.raw(`https://ungh.cc/repos/${shortName}/${shortName}`).catch(() => null))?.ok) return `https://github.com/${shortName}/${shortName}`;
1765
+ const SKIP_DIRS = [
1766
+ "node_modules",
1767
+ "_vendor",
1768
+ "__tests__",
1769
+ "__mocks__",
1770
+ "__fixtures__",
1771
+ "test",
1772
+ "tests",
1773
+ "fixture",
1774
+ "fixtures",
1775
+ "locales",
1776
+ "locale",
1777
+ "i18n",
1778
+ ".git"
1779
+ ];
1780
+ const SKIP_PATTERNS = [
1781
+ "*.min.*",
1782
+ "*.prod.*",
1783
+ "*.global.*",
1784
+ "*.browser.*",
1785
+ "*.map",
1786
+ "*.map.js",
1787
+ "CHANGELOG*",
1788
+ "LICENSE*",
1789
+ "README*"
1790
+ ];
1791
+ const MAX_FILE_SIZE = 500 * 1024;
1792
+ async function resolveEntryFiles(packageDir) {
1793
+ if (!existsSync(join(packageDir, "package.json"))) return [];
1794
+ const files = await glob(["**/*.d.{ts,mts,cts}"], {
1795
+ cwd: packageDir,
1796
+ ignore: [...SKIP_DIRS.map((d) => `**/${d}/**`), ...SKIP_PATTERNS],
1797
+ absolute: false,
1798
+ expandDirectories: false
1799
+ });
1800
+ const entries = [];
1801
+ for (const file of files) {
1802
+ const absPath = join(packageDir, file);
1803
+ let content;
1804
+ try {
1805
+ content = readFileSync(absPath, "utf-8");
1806
+ } catch {
1720
1807
  continue;
1721
1808
  }
1722
- if ((await $fetch.raw(`https://ungh.cc/repos/${candidate}`).catch(() => null))?.ok) return `https://github.com/${candidate}`;
1723
- }
1724
- const searchTerm = packageName.replace(/^@/, "");
1725
- if (isGhAvailable()) try {
1726
- const { stdout: json } = spawnSync("gh", [
1727
- "search",
1728
- "repos",
1729
- searchTerm,
1730
- "--json",
1731
- "fullName",
1732
- "--limit",
1733
- "5"
1734
- ], {
1735
- encoding: "utf-8",
1736
- timeout: 15e3
1809
+ if (content.length > MAX_FILE_SIZE) continue;
1810
+ entries.push({
1811
+ path: file,
1812
+ content,
1813
+ type: "types"
1737
1814
  });
1738
- if (!json) throw new Error("no output");
1739
- const repos = JSON.parse(json);
1740
- const match = repos.find((r) => r.fullName.toLowerCase().endsWith(`/${packageName.toLowerCase()}`) || r.fullName.toLowerCase().endsWith(`/${shortName.toLowerCase()}`));
1741
- if (match) return `https://github.com/${match.fullName}`;
1742
- for (const candidate of repos) {
1743
- const gh = parseGitHubUrl(`https://github.com/${candidate.fullName}`);
1744
- if (gh && await verifyNpmRepo(gh.owner, gh.repo, packageName)) return `https://github.com/${candidate.fullName}`;
1815
+ }
1816
+ return entries;
1817
+ }
1818
+ function parseGitSkillInput(input) {
1819
+ const trimmed = input.trim();
1820
+ if (trimmed.startsWith("@")) return null;
1821
+ if (trimmed.startsWith("./") || trimmed.startsWith("../") || trimmed.startsWith("/") || trimmed.startsWith("~")) return {
1822
+ type: "local",
1823
+ localPath: trimmed.startsWith("~") ? resolve(process.env.HOME || "", trimmed.slice(1)) : resolve(trimmed)
1824
+ };
1825
+ if (trimmed.startsWith("git@")) {
1826
+ const gh = parseGitHubUrl(normalizeRepoUrl(trimmed));
1827
+ if (gh) return {
1828
+ type: "github",
1829
+ owner: gh.owner,
1830
+ repo: gh.repo
1831
+ };
1832
+ return null;
1833
+ }
1834
+ if (trimmed.startsWith("https://") || trimmed.startsWith("http://")) return parseGitUrl(trimmed);
1835
+ if (/^[\w.-]+\/[\w.-]+$/.test(trimmed)) return {
1836
+ type: "github",
1837
+ owner: trimmed.split("/")[0],
1838
+ repo: trimmed.split("/")[1]
1839
+ };
1840
+ return null;
1841
+ }
1842
+ function parseGitUrl(url) {
1843
+ try {
1844
+ const parsed = new URL(url);
1845
+ if (parsed.hostname === "github.com" || parsed.hostname === "www.github.com") {
1846
+ const parts = parsed.pathname.replace(/^\//, "").replace(/\.git$/, "").split("/");
1847
+ const owner = parts[0];
1848
+ const repo = parts[1];
1849
+ if (!owner || !repo) return null;
1850
+ if (parts[2] === "tree" && parts.length >= 4) return {
1851
+ type: "github",
1852
+ owner,
1853
+ repo,
1854
+ ref: parts[3],
1855
+ skillPath: parts.length > 4 ? parts.slice(4).join("/") : void 0
1856
+ };
1857
+ return {
1858
+ type: "github",
1859
+ owner,
1860
+ repo
1861
+ };
1745
1862
  }
1746
- } catch {}
1747
- const data = await $fetch(`https://api.github.com/search/repositories?q=${encodeURIComponent(`${searchTerm} in:name`)}&per_page=5`).catch(() => null);
1748
- if (!data?.items?.length) return null;
1749
- const match = data.items.find((r) => r.full_name.toLowerCase().endsWith(`/${packageName.toLowerCase()}`) || r.full_name.toLowerCase().endsWith(`/${shortName.toLowerCase()}`));
1750
- if (match) return `https://github.com/${match.full_name}`;
1751
- for (const candidate of data.items) {
1752
- const gh = parseGitHubUrl(`https://github.com/${candidate.full_name}`);
1753
- if (gh && await verifyNpmRepo(gh.owner, gh.repo, packageName)) return `https://github.com/${candidate.full_name}`;
1863
+ if (parsed.hostname === "gitlab.com") {
1864
+ const parts = parsed.pathname.replace(/^\//, "").replace(/\.git$/, "").split("/");
1865
+ const owner = parts[0];
1866
+ const repo = parts[1];
1867
+ if (!owner || !repo) return null;
1868
+ return {
1869
+ type: "gitlab",
1870
+ owner,
1871
+ repo
1872
+ };
1873
+ }
1874
+ return null;
1875
+ } catch {
1876
+ return null;
1754
1877
  }
1755
- return null;
1756
1878
  }
1757
- async function fetchGitHubRepoMeta(owner, repo, packageName) {
1758
- const override = packageName ? getDocOverride(packageName) : void 0;
1759
- if (override?.homepage) return { homepage: override.homepage };
1760
- const data = await ghApi(`repos/${owner}/${repo}`) ?? await $fetch(`https://api.github.com/repos/${owner}/${repo}`).catch(() => null);
1761
- return data?.homepage ? { homepage: data.homepage } : null;
1879
+ function parseSkillFrontmatterName(content) {
1880
+ const fm = parseFrontmatter(content);
1881
+ return {
1882
+ name: fm.name,
1883
+ description: fm.description
1884
+ };
1762
1885
  }
1763
- async function fetchReadme(owner, repo, subdir, ref) {
1764
- const branch = ref || "main";
1765
- if (!isKnownPrivateRepo(owner, repo)) {
1766
- const unghUrl = subdir ? `https://ungh.cc/repos/${owner}/${repo}/files/${branch}/${subdir}/README.md` : `https://ungh.cc/repos/${owner}/${repo}/readme${ref ? `?ref=${ref}` : ""}`;
1767
- if ((await $fetch.raw(unghUrl).catch(() => null))?.ok) return `ungh://${owner}/${repo}${subdir ? `/${subdir}` : ""}${ref ? `@${ref}` : ""}`;
1886
+ function collectFiles(dir, prefix = "") {
1887
+ const files = [];
1888
+ if (!existsSync(dir)) return files;
1889
+ for (const entry of readdirSync(dir, { withFileTypes: true })) {
1890
+ const relPath = prefix ? `${prefix}/${entry.name}` : entry.name;
1891
+ const fullPath = resolve(dir, entry.name);
1892
+ if (entry.isDirectory()) files.push(...collectFiles(fullPath, relPath));
1893
+ else if (entry.isFile()) files.push({
1894
+ path: relPath,
1895
+ content: readFileSync(fullPath, "utf-8")
1896
+ });
1768
1897
  }
1769
- const basePath = subdir ? `${subdir}/` : "";
1770
- const branches = ref ? [ref] : ["main", "master"];
1771
- const token = isKnownPrivateRepo(owner, repo) ? getGitHubToken() : null;
1772
- const authHeaders = token ? { Authorization: `token ${token}` } : {};
1773
- for (const b of branches) for (const filename of [
1774
- "README.md",
1775
- "Readme.md",
1776
- "readme.md"
1777
- ]) {
1778
- const readmeUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${b}/${basePath}${filename}`;
1779
- if ((await $fetch.raw(readmeUrl, { headers: authHeaders }).catch(() => null))?.ok) return readmeUrl;
1898
+ return files;
1899
+ }
1900
+ async function fetchGitSkills(source, onProgress) {
1901
+ if (source.type === "local") return fetchLocalSkills(source);
1902
+ if (source.type === "github") return fetchGitHubSkills(source, onProgress);
1903
+ if (source.type === "gitlab") return fetchGitLabSkills(source, onProgress);
1904
+ return { skills: [] };
1905
+ }
1906
+ function fetchLocalSkills(source) {
1907
+ const base = source.localPath;
1908
+ if (!existsSync(base)) return { skills: [] };
1909
+ const skills = [];
1910
+ const skillsDir = resolve(base, "skills");
1911
+ if (existsSync(skillsDir)) for (const entry of readdirSync(skillsDir, { withFileTypes: true })) {
1912
+ if (!entry.isDirectory()) continue;
1913
+ const skill = readLocalSkill(resolve(skillsDir, entry.name), `skills/${entry.name}`);
1914
+ if (skill) skills.push(skill);
1780
1915
  }
1781
- const refParam = ref ? `?ref=${ref}` : "";
1782
- const apiData = await ghApi(subdir ? `repos/${owner}/${repo}/contents/${subdir}/README.md${refParam}` : `repos/${owner}/${repo}/readme${refParam}`);
1783
- if (apiData?.download_url) {
1784
- markRepoPrivate(owner, repo);
1785
- return apiData.download_url;
1916
+ if (skills.length === 0) {
1917
+ const skill = readLocalSkill(base, "");
1918
+ if (skill) skills.push(skill);
1786
1919
  }
1787
- return null;
1920
+ return { skills };
1788
1921
  }
1789
- async function fetchReadmeContent(url) {
1790
- if (url.startsWith("file://")) {
1791
- const filePath = fileURLToPath(url);
1792
- if (!existsSync(filePath)) return null;
1793
- return readFileSync(filePath, "utf-8");
1922
+ function readLocalSkill(dir, repoPath) {
1923
+ const skillMdPath = resolve(dir, "SKILL.md");
1924
+ if (!existsSync(skillMdPath)) return null;
1925
+ const content = readFileSync(skillMdPath, "utf-8");
1926
+ const frontmatter = parseSkillFrontmatterName(content);
1927
+ const dirName = dir.split("/").pop();
1928
+ const name = frontmatter.name || dirName;
1929
+ const files = collectFiles(dir).filter((f) => f.path !== "SKILL.md");
1930
+ return {
1931
+ name,
1932
+ description: frontmatter.description || "",
1933
+ path: repoPath,
1934
+ content,
1935
+ files
1936
+ };
1937
+ }
1938
+ async function fetchGitHubSkills(source, onProgress) {
1939
+ const { owner, repo } = source;
1940
+ if (!owner || !repo) return { skills: [] };
1941
+ const ref = source.ref || "main";
1942
+ const refs = ref === "main" ? ["main", "master"] : [ref];
1943
+ for (const tryRef of refs) {
1944
+ const skills = await downloadGitHubSkills(owner, repo, tryRef, source.skillPath, onProgress);
1945
+ if (skills.length > 0) return { skills };
1794
1946
  }
1795
- if (url.startsWith("ungh://")) {
1796
- let path = url.replace("ungh://", "");
1797
- let ref = "main";
1798
- const atIdx = path.lastIndexOf("@");
1799
- if (atIdx !== -1) {
1800
- ref = path.slice(atIdx + 1);
1801
- path = path.slice(0, atIdx);
1947
+ return { skills: [] };
1948
+ }
1949
+ async function downloadGitHubSkills(owner, repo, ref, skillPath, onProgress) {
1950
+ const tempDir = join(tmpdir(), `skilld-${Date.now()}`);
1951
+ try {
1952
+ if (skillPath) {
1953
+ onProgress?.(`Downloading ${owner}/${repo}/${skillPath}@${ref}`);
1954
+ const { dir } = await downloadTemplate(`github:${owner}/${repo}/${skillPath}#${ref}`, {
1955
+ dir: tempDir,
1956
+ force: true,
1957
+ auth: getGitHubToken() || void 0
1958
+ });
1959
+ const skill = readLocalSkill(dir, skillPath);
1960
+ return skill ? [skill] : [];
1802
1961
  }
1803
- const parts = path.split("/");
1804
- const owner = parts[0];
1805
- const repo = parts[1];
1806
- const subdir = parts.slice(2).join("/");
1807
- const text = await $fetch(subdir ? `https://ungh.cc/repos/${owner}/${repo}/files/${ref}/${subdir}/README.md` : `https://ungh.cc/repos/${owner}/${repo}/readme?ref=${ref}`, { responseType: "text" }).catch(() => null);
1808
- if (!text) return null;
1962
+ onProgress?.(`Downloading ${owner}/${repo}/skills@${ref}`);
1809
1963
  try {
1810
- const json = JSON.parse(text);
1811
- return json.markdown || json.file?.contents || null;
1812
- } catch {
1813
- return text;
1964
+ const { dir } = await downloadTemplate(`github:${owner}/${repo}/skills#${ref}`, {
1965
+ dir: tempDir,
1966
+ force: true,
1967
+ auth: getGitHubToken() || void 0
1968
+ });
1969
+ const skills = [];
1970
+ for (const entry of readdirSync(dir, { withFileTypes: true })) {
1971
+ if (!entry.isDirectory()) continue;
1972
+ const skill = readLocalSkill(resolve(dir, entry.name), `skills/${entry.name}`);
1973
+ if (skill) skills.push(skill);
1974
+ }
1975
+ if (skills.length > 0) {
1976
+ onProgress?.(`Found ${skills.length} skill(s)`);
1977
+ return skills;
1978
+ }
1979
+ } catch {}
1980
+ const content = await fetchGitHubRaw(`https://raw.githubusercontent.com/${owner}/${repo}/${ref}/SKILL.md`);
1981
+ if (content) {
1982
+ const fm = parseSkillFrontmatterName(content);
1983
+ onProgress?.("Found 1 skill");
1984
+ return [{
1985
+ name: fm.name || repo,
1986
+ description: fm.description || "",
1987
+ path: "",
1988
+ content,
1989
+ files: []
1990
+ }];
1814
1991
  }
1992
+ return [];
1993
+ } catch {
1994
+ return [];
1995
+ } finally {
1996
+ rmSync(tempDir, {
1997
+ recursive: true,
1998
+ force: true
1999
+ });
1815
2000
  }
1816
- if (url.includes("raw.githubusercontent.com")) return fetchGitHubRaw(url);
1817
- return fetchText(url);
1818
2001
  }
1819
- async function resolveGitHubRepo(owner, repo, onProgress) {
1820
- onProgress?.("Fetching repo metadata");
1821
- const repoUrl = `https://github.com/${owner}/${repo}`;
1822
- const meta = await ghApi(`repos/${owner}/${repo}`) ?? await $fetch(`https://api.github.com/repos/${owner}/${repo}`).catch(() => null);
1823
- const homepage = meta?.homepage || void 0;
1824
- const description = meta?.description || void 0;
1825
- onProgress?.("Fetching latest release");
1826
- const releases = await fetchUnghReleases(owner, repo);
1827
- let version = "main";
1828
- let releasedAt;
1829
- const latestRelease = releases[0];
1830
- if (latestRelease) {
1831
- version = latestRelease.tag.replace(/^v/, "");
1832
- releasedAt = latestRelease.publishedAt;
1833
- }
1834
- onProgress?.("Resolving docs");
1835
- const gitDocs = await fetchGitDocs(owner, repo, version);
1836
- const gitDocsUrl = gitDocs ? `${repoUrl}/tree/${gitDocs.ref}/docs` : void 0;
1837
- const gitRef = gitDocs?.ref;
1838
- onProgress?.("Fetching README");
1839
- const readmeUrl = await fetchReadme(owner, repo);
1840
- let llmsUrl;
1841
- if (homepage) {
1842
- onProgress?.("Checking llms.txt");
1843
- llmsUrl = await fetchLlmsUrl(homepage).catch(() => null) ?? void 0;
2002
+ async function fetchGitLabSkills(source, onProgress) {
2003
+ const { owner, repo } = source;
2004
+ if (!owner || !repo) return { skills: [] };
2005
+ const ref = source.ref || "main";
2006
+ const tempDir = join(tmpdir(), `skilld-gitlab-${Date.now()}`);
2007
+ try {
2008
+ const subdir = source.skillPath || "skills";
2009
+ onProgress?.(`Downloading ${owner}/${repo}/${subdir}@${ref}`);
2010
+ const { dir } = await downloadTemplate(`gitlab:${owner}/${repo}/${subdir}#${ref}`, {
2011
+ dir: tempDir,
2012
+ force: true
2013
+ });
2014
+ if (source.skillPath) {
2015
+ const skill = readLocalSkill(dir, source.skillPath);
2016
+ return { skills: skill ? [skill] : [] };
2017
+ }
2018
+ const skills = [];
2019
+ for (const entry of readdirSync(dir, { withFileTypes: true })) {
2020
+ if (!entry.isDirectory()) continue;
2021
+ const skill = readLocalSkill(resolve(dir, entry.name), `skills/${entry.name}`);
2022
+ if (skill) skills.push(skill);
2023
+ }
2024
+ if (skills.length > 0) {
2025
+ onProgress?.(`Found ${skills.length} skill(s)`);
2026
+ return { skills };
2027
+ }
2028
+ const content = await $fetch(`https://gitlab.com/${owner}/${repo}/-/raw/${ref}/SKILL.md`, { responseType: "text" }).catch(() => null);
2029
+ if (content) {
2030
+ const fm = parseSkillFrontmatterName(content);
2031
+ return { skills: [{
2032
+ name: fm.name || repo,
2033
+ description: fm.description || "",
2034
+ path: "",
2035
+ content,
2036
+ files: []
2037
+ }] };
2038
+ }
2039
+ return { skills: [] };
2040
+ } catch {
2041
+ return { skills: [] };
2042
+ } finally {
2043
+ rmSync(tempDir, {
2044
+ recursive: true,
2045
+ force: true
2046
+ });
1844
2047
  }
1845
- if (!gitDocsUrl && !readmeUrl && !llmsUrl) return null;
1846
- return {
1847
- name: repo,
1848
- version: latestRelease ? version : void 0,
1849
- releasedAt,
1850
- description,
1851
- repoUrl,
1852
- docsUrl: homepage,
1853
- gitDocsUrl,
1854
- gitRef,
1855
- gitDocsFallback: gitDocs?.fallback,
1856
- readmeUrl: readmeUrl ?? void 0,
1857
- llmsUrl
1858
- };
1859
2048
  }
1860
2049
  async function searchNpmPackages(query, size = 5) {
1861
2050
  const data = await $fetch(`https://registry.npmjs.org/-/v1/search?text=${encodeURIComponent(query)}&size=${size}`).catch(() => null);
@@ -2198,7 +2387,7 @@ async function fetchPkgDist(name, version) {
2198
2387
  if (!data) return null;
2199
2388
  const tarballUrl = data.dist?.tarball;
2200
2389
  if (!tarballUrl) return null;
2201
- const tarballRes = await fetch(tarballUrl, { headers: { "User-Agent": "skilld/1.0" } }).catch(() => null);
2390
+ const tarballRes = await fetch(tarballUrl, { headers: { "User-Agent": SKILLD_USER_AGENT } }).catch(() => null);
2202
2391
  if (!tarballRes?.ok || !tarballRes.body) return null;
2203
2392
  mkdirSync(pkgDir, { recursive: true });
2204
2393
  const tmpTarball = join(cacheDir, "_pkg.tgz");
@@ -2267,6 +2456,6 @@ function getInstalledSkillVersion(skillDir) {
2267
2456
  if (!existsSync(skillPath)) return null;
2268
2457
  return readFileSync(skillPath, "utf-8").match(/^version:\s*"?([^"\n]+)"?/m)?.[1] || null;
2269
2458
  }
2270
- export { isGitHubRepoUrl as $, parseGitSkillInput as A, isGhAvailable as B, downloadLlmsDocs as C, normalizeLlmsLinks as D, fetchLlmsUrl as E, formatDiscussionAsMarkdown as F, fetchReleaseNotes as G, toCrawlPattern as H, generateDiscussionIndex as I, parseSemver as J, generateReleaseIndex as K, fetchGitHubIssues as L, resolveEntryFiles as M, generateDocsIndex as N, parseMarkdownLinks as O, fetchGitHubDiscussions as P, fetchText as Q, formatIssueAsMarkdown as R, validateGitDocsWithLlms as S, fetchLlmsTxt as T, fetchBlogReleases as U, fetchCrawledDocs as V, compareSemver as W, extractBranchHint as X, $fetch as Y, fetchGitHubRaw as Z, fetchReadme as _, getInstalledSkillVersion as a, isShallowGitDocs as b, readLocalPackageInfo as c, resolvePackageDocs as d, normalizeRepoUrl as et, resolvePackageDocsWithAttempts as f, fetchGitHubRepoMeta as g, fetchGitDocs as h, fetchPkgDist as i, parseSkillFrontmatterName as j, fetchGitSkills as k, resolveInstalledVersion as l, MIN_GIT_DOCS as m, fetchNpmPackage as n, parsePackageSpec as nt, parseVersionSpecifier as o, searchNpmPackages as p, isPrerelease as q, fetchNpmRegistryMeta as r, verifyUrl as rt, readLocalDependencies as s, fetchLatestVersion as t, parseGitHubUrl as tt, resolveLocalPackageDocs as u, fetchReadmeContent as v, extractSections as w, resolveGitHubRepo as x, filterFrameworkDocs as y, generateIssueIndex as z };
2459
+ export { fetchText as $, filterFrameworkDocs as A, fetchGitHubIssues as B, toCrawlPattern as C, fetchGitHubRepoMeta as D, fetchGitDocs as E, extractSections as F, compareSemver as G, generateIssueIndex as H, fetchLlmsTxt as I, isPrerelease as J, fetchReleaseNotes as K, fetchLlmsUrl as L, resolveGitHubRepo as M, validateGitDocsWithLlms as N, fetchReadme as O, downloadLlmsDocs as P, fetchGitHubRaw as Q, normalizeLlmsLinks as R, fetchCrawledDocs as S, MIN_GIT_DOCS as T, isGhAvailable as U, formatIssueAsMarkdown as V, fetchBlogReleases as W, $fetch as X, parseSemver as Y, extractBranchHint as Z, resolveEntryFiles as _, getInstalledSkillVersion as a, formatDiscussionAsMarkdown as b, readLocalPackageInfo as c, resolvePackageDocs as d, isGitHubRepoUrl as et, resolvePackageDocsWithAttempts as f, parseSkillFrontmatterName as g, parseGitSkillInput as h, fetchPkgDist as i, verifyUrl as it, isShallowGitDocs as j, fetchReadmeContent as k, resolveInstalledVersion as l, fetchGitSkills as m, fetchNpmPackage as n, parseGitHubUrl as nt, parseVersionSpecifier as o, searchNpmPackages as p, generateReleaseIndex as q, fetchNpmRegistryMeta as r, parsePackageSpec as rt, readLocalDependencies as s, fetchLatestVersion as t, normalizeRepoUrl as tt, resolveLocalPackageDocs as u, generateDocsIndex as v, resolveCrateDocsWithAttempts as w, generateDiscussionIndex as x, fetchGitHubDiscussions as y, parseMarkdownLinks as z };
2271
2460
 
2272
2461
  //# sourceMappingURL=sources.mjs.map