skilld 0.11.3 → 0.12.1

package/dist/_chunks/npm.mjs

@@ -1,12 +1,14 @@
-import { i as getCacheDir } from "./config.mjs";
+import { o as getCacheDir } from "./config.mjs";
 import { i as parseFrontmatter, n as extractLinks, r as extractTitle, t as extractDescription } from "./markdown.mjs";
-import { n as getDocOverride, t as getBlogPreset } from "./package-registry.mjs";
+import { n as getCrawlUrl, r as getDocOverride, t as getBlogPreset } from "./package-registry.mjs";
+import { tmpdir } from "node:os";
 import { basename, dirname, join, resolve } from "pathe";
 import { createWriteStream, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, unlinkSync } from "node:fs";
 import { htmlToMarkdown } from "mdream";
 import { spawnSync } from "node:child_process";
 import { gt } from "semver";
 import { ofetch } from "ofetch";
+import { crawlAndGenerate } from "@mdream/crawl";
 import { globby } from "globby";
 import pLimit from "p-limit";
 import { fileURLToPath, pathToFileURL } from "node:url";
@@ -722,6 +724,38 @@ async function fetchBlogReleases(packageName, installedVersion) {
 		content: formatBlogRelease(r)
 	}));
 }
+async function fetchCrawledDocs(url, onProgress, maxPages = 200) {
+	const outputDir = join(tmpdir(), "skilld-crawl", Date.now().toString());
+	onProgress?.(`Crawling ${url}`);
+	const results = await crawlAndGenerate({
+		urls: [url],
+		outputDir,
+		driver: "http",
+		generateLlmsTxt: false,
+		generateIndividualMd: false,
+		maxRequestsPerCrawl: maxPages
+	}, (progress) => {
+		if (progress.crawling.status === "processing" && progress.crawling.total > 0) onProgress?.(`Crawling ${progress.crawling.processed}/${progress.crawling.total} pages`);
+	});
+	rmSync(outputDir, {
+		recursive: true,
+		force: true
+	});
+	const docs = [];
+	for (const result of results) {
+		if (!result.success || !result.content) continue;
+		const path = `docs/${(new URL(result.url).pathname.replace(/\/$/, "") || "/index").split("/").filter(Boolean).join("/")}.md`;
+		docs.push({
+			path,
+			content: result.content
+		});
+	}
+	onProgress?.(`Crawled ${docs.length} pages`);
+	return docs;
+}
+function toCrawlPattern(docsUrl) {
+	return `${docsUrl.replace(/\/+$/, "")}/**`;
+}
 const HIGH_VALUE_CATEGORIES = new Set([
 	"q&a",
 	"help",
@@ -1372,6 +1406,27 @@ async function findLatestReleaseTag(owner, repo, packageName) {
 function filterDocFiles(files, pathPrefix) {
 	return files.filter((f) => f.startsWith(pathPrefix) && /\.(?:md|mdx)$/.test(f));
 }
+const FRAMEWORK_NAMES = new Set([
+	"vue",
+	"react",
+	"solid",
+	"angular",
+	"svelte",
+	"preact",
+	"lit",
+	"qwik"
+]);
+function filterFrameworkDocs(files, packageName) {
+	if (!packageName) return files;
+	const shortName = packageName.replace(/^@.*\//, "");
+	const targetFramework = [...FRAMEWORK_NAMES].find((fw) => shortName.includes(fw));
+	if (!targetFramework) return files;
+	const frameworkPattern = new RegExp(`(?:^|/)(?:framework/)?(?:${[...FRAMEWORK_NAMES].join("|")})/`);
+	if (!files.some((f) => frameworkPattern.test(f))) return files;
+	const otherFrameworks = [...FRAMEWORK_NAMES].filter((fw) => fw !== targetFramework);
+	const excludePattern = new RegExp(`(?:^|/)(?:framework/)?(?:${otherFrameworks.join("|")})/`);
+	return files.filter((f) => !excludePattern.test(f));
+}
 const NOISE_PATTERNS = [
 	/^\.changeset\//,
 	/CHANGELOG\.md$/i,
@@ -1421,8 +1476,16 @@ function scoreDocDir(dir, fileCount) {
 	const depth = getPathDepth(dir) || 1;
 	return fileCount * (hasDocDirBonus(dir) ? 1.5 : 1) / depth;
 }
-function discoverDocFiles(allFiles) {
+function discoverDocFiles(allFiles, packageName) {
 	const mdFiles = allFiles.filter((f) => /\.(?:md|mdx)$/.test(f)).filter((f) => !NOISE_PATTERNS.some((p) => p.test(f))).filter((f) => f.includes("/"));
+	if (packageName?.includes("/")) {
+		const subPkgPrefix = `packages/${packageName.split("/").pop().toLowerCase()}/`;
+		const subPkgFiles = mdFiles.filter((f) => f.startsWith(subPkgPrefix));
+		if (subPkgFiles.length >= 3) return {
+			files: subPkgFiles,
+			prefix: subPkgPrefix
+		};
+	}
 	const docsGroups = /* @__PURE__ */ new Map();
 	for (const file of mdFiles) {
 		const docsIdx = file.lastIndexOf("/docs/");
@@ -1485,13 +1548,14 @@ async function fetchGitDocs(owner, repo, version, packageName, repoUrl) {
 	let docsPrefix;
 	let allFiles;
 	if (docs.length === 0) {
-		const discovered = discoverDocFiles(tag.files);
+		const discovered = discoverDocFiles(tag.files, packageName);
 		if (discovered) {
 			docs = discovered.files;
 			docsPrefix = discovered.prefix || void 0;
 			allFiles = tag.files;
 		}
 	}
+	docs = filterFrameworkDocs(docs, packageName);
 	if (docs.length === 0) return null;
 	return {
 		baseUrl: `https://raw.githubusercontent.com/${owner}/${repo}/${tag.ref}`,
@@ -1875,6 +1939,8 @@ async function resolvePackageDocsWithAttempts(packageName, options = {}) {
 			message: "No repository URL in package.json and GitHub search found no match"
 		});
 	}
+	const crawlUrl = getCrawlUrl(packageName);
+	if (crawlUrl) result.crawlUrl = crawlUrl;
 	if (result.docsUrl) {
 		onProgress?.("llms.txt");
 		const llmsUrl = await fetchLlmsUrl(result.docsUrl);
@@ -2104,6 +2170,6 @@ function getInstalledSkillVersion(skillDir) {
 	if (!existsSync(skillPath)) return null;
 	return readFileSync(skillPath, "utf-8").match(/^version:\s*"?([^"\n]+)"?/m)?.[1] || null;
 }
-export { isGhAvailable as $, parseSkillFrontmatterName as A, isPrerelease as B, extractSections as C, parseMarkdownLinks as D, normalizeLlmsLinks as E, generateDiscussionIndex as F, isGitHubRepoUrl as G, $fetch as H, fetchBlogReleases as I, parsePackageSpec as J, normalizeRepoUrl as K, compareSemver as L, generateDocsIndex as M, fetchGitHubDiscussions as N, fetchGitSkills as O, formatDiscussionAsMarkdown as P, generateIssueIndex as Q, fetchReleaseNotes as R, downloadLlmsDocs as S, fetchLlmsUrl as T, extractBranchHint as U, parseSemver as V, fetchText as W, fetchGitHubIssues as X, verifyUrl as Y, formatIssueAsMarkdown as Z, fetchReadme as _, getInstalledSkillVersion as a, resolveGitHubRepo as b, readLocalPackageInfo as c, resolvePackageDocs as d, SHARED_SKILLS_DIR as et, resolvePackageDocsWithAttempts as f, fetchGitHubRepoMeta as g, fetchGitDocs as h, fetchPkgDist as i, resolveEntryFiles as j, parseGitSkillInput as k, resolveInstalledVersion as l, MIN_GIT_DOCS as m, fetchNpmPackage as n, mapInsert as nt, parseVersionSpecifier as o, searchNpmPackages as p, parseGitHubUrl as q, fetchNpmRegistryMeta as r, semverGt as rt, readLocalDependencies as s, fetchLatestVersion as t, getSharedSkillsDir as tt, resolveLocalPackageDocs as u, fetchReadmeContent as v, fetchLlmsTxt as w, validateGitDocsWithLlms as x, isShallowGitDocs as y, generateReleaseIndex as z };
+export { formatIssueAsMarkdown as $, parseSkillFrontmatterName as A, fetchReleaseNotes as B, extractSections as C, parseMarkdownLinks as D, normalizeLlmsLinks as E, generateDiscussionIndex as F, extractBranchHint as G, isPrerelease as H, fetchCrawledDocs as I, normalizeRepoUrl as J, fetchText as K, toCrawlPattern as L, generateDocsIndex as M, fetchGitHubDiscussions as N, fetchGitSkills as O, formatDiscussionAsMarkdown as P, fetchGitHubIssues as Q, fetchBlogReleases as R, downloadLlmsDocs as S, fetchLlmsUrl as T, parseSemver as U, generateReleaseIndex as V, $fetch as W, parsePackageSpec as X, parseGitHubUrl as Y, verifyUrl as Z, fetchReadme as _, getInstalledSkillVersion as a, semverGt as at, resolveGitHubRepo as b, readLocalPackageInfo as c, resolvePackageDocs as d, generateIssueIndex as et, resolvePackageDocsWithAttempts as f, fetchGitHubRepoMeta as g, fetchGitDocs as h, fetchPkgDist as i, mapInsert as it, resolveEntryFiles as j, parseGitSkillInput as k, resolveInstalledVersion as l, MIN_GIT_DOCS as m, fetchNpmPackage as n, SHARED_SKILLS_DIR as nt, parseVersionSpecifier as o, searchNpmPackages as p, isGitHubRepoUrl as q, fetchNpmRegistryMeta as r, getSharedSkillsDir as rt, readLocalDependencies as s, fetchLatestVersion as t, isGhAvailable as tt, resolveLocalPackageDocs as u, fetchReadmeContent as v, fetchLlmsTxt as w, validateGitDocsWithLlms as x, isShallowGitDocs as y, compareSemver as z };
 
 //# sourceMappingURL=npm.mjs.map