rspress-plugin-api-extractor 0.1.2 → 0.2.1

package/llms-processing.js

@@ -0,0 +1,270 @@
+//#region src/llms-processing.ts
+/** Pre-compiled regex for parsing llms.txt link lines. */
+const LLMS_TXT_LINE_RE = /^-\s+\[([^\]]+)\]\(([^)]+)\)(?::\s*(.+))?$/;
+/**
+* Parse a single line from llms.txt format.
+*
+* Recognizes the pattern: `- [title](url): description`
+* The description portion (`: description`) is optional.
+*
+* @param line - A single line from an llms.txt file
+* @returns Parsed entry or null for non-link lines (headers, empty lines, plain text)
+*/
+function parseLlmsTxtLine(line) {
+	const trimmed = line.trim();
+	if (trimmed === "") return null;
+	const match = LLMS_TXT_LINE_RE.exec(trimmed);
+	if (!match) return null;
+	const title = match[1];
+	const url = match[2];
+	const rawDescription = match[3];
+	return {
+		title,
+		url,
+		description: rawDescription ? rawDescription.trim() : void 0
+	};
+}
+/**
+* Filter API page entries from global llms.txt content.
+*
+* Removes lines whose parsed URL is in the `apiRoutes` set.
+* Appends pointer lines for per-package llms files when `pointers` is non-empty.
+*
+* @param content - Full llms.txt content string
+* @param apiRoutes - Set of API route paths to remove
+* @param pointers - Per-package pointer entries to append
+* @returns Filtered llms.txt content
+*/
+function filterLlmsTxt(content, apiRoutes, pointers) {
+	const lines = content.split("\n");
+	const filtered = [];
+	for (const line of lines) {
+		const entry = parseLlmsTxtLine(line);
+		if (entry && apiRoutes.has(entry.url)) continue;
+		filtered.push(line);
+	}
+	let result = filtered.join("\n");
+	if (pointers.length > 0) {
+		result += "\n\n";
+		for (const pointer of pointers) result += `- For ${pointer.name} API docs, see [${pointer.name} llms.txt](${pointer.llmsTxtUrl})\n`;
+	}
+	return result;
+}
+/**
+* Generate a structured global llms.txt that groups pages by package scope.
+*
+* Output format:
+* ```
+* # {site title}
+*
+* ## Others
+* - [Blog Post](/blog/post.md)
+*
+* ## Packages
+*
+* ### {name} {version}
+* {description}
+* - [Guide Page](/pkg/guides/guide.md)
+* - [API Reference](/pkg/llms-api.txt)
+* ```
+*
+* @param content - Original RSPress-generated llms.txt content
+* @param apiRoutes - Set of API route paths to exclude as individual entries
+* @param packages - Package scope metadata
+* @returns Restructured llms.txt content
+*/
+function generateStructuredLlmsTxt(content, apiRoutes, packages) {
+	const lines = content.split("\n");
+	let title = "";
+	for (const line of lines) if (line.startsWith("# ")) {
+		title = line;
+		break;
+	}
+	const allEntries = [];
+	for (const line of lines) {
+		const entry = parseLlmsTxtLine(line);
+		if (entry && !apiRoutes.has(entry.url)) allEntries.push(entry);
+	}
+	const packageEntries = /* @__PURE__ */ new Map();
+	const others = [];
+	for (const entry of allEntries) {
+		let matched = false;
+		for (const pkg of packages) {
+			const base = pkg.packageRoute.endsWith("/") ? pkg.packageRoute : `${pkg.packageRoute}/`;
+			if (entry.url.startsWith(base) || entry.url === pkg.packageRoute) {
+				const existing = packageEntries.get(pkg.packageName) ?? [];
+				existing.push(entry);
+				packageEntries.set(pkg.packageName, existing);
+				matched = true;
+				break;
+			}
+		}
+		if (!matched) others.push(entry);
+	}
+	const output = [];
+	if (title) {
+		output.push(title);
+		output.push("");
+	}
+	if (others.length > 0) {
+		output.push("## Others");
+		output.push("");
+		for (const entry of others) output.push(formatEntry(entry));
+		output.push("");
+	}
+	const packagesWithEntries = packages;
+	if (packagesWithEntries.length > 0) {
+		output.push("## Packages");
+		output.push("");
+		for (const pkg of packagesWithEntries) {
+			const versionSuffix = pkg.version ? ` ${pkg.version}` : "";
+			output.push(`### ${pkg.name}${versionSuffix}`);
+			output.push("");
+			if (pkg.description) {
+				output.push(pkg.description);
+				output.push("");
+			}
+			const entries = packageEntries.get(pkg.packageName) ?? [];
+			for (const entry of entries) output.push(formatEntry(entry));
+			output.push(`- [API Reference](${pkg.llmsApiTxtUrl})`);
+			output.push("");
+		}
+	}
+	return output.join("\n");
+}
+/**
+* Parse llms-full.txt content into sections delimited by frontmatter blocks.
+*
+* Each section has the format:
+* ```
+* ---
+* url: /path/to/page
+* ---
+*
+* Content here...
+* ```
+*/
+function parseSections(content) {
+	if (content.trim() === "") return [];
+	const sections = [];
+	const frontmatterPattern = /^---\nurl:\s*(.+)\n---$/gm;
+	let match = frontmatterPattern.exec(content);
+	const boundaries = [];
+	while (match !== null) {
+		boundaries.push({
+			url: match[1].trim(),
+			start: match.index,
+			fmEnd: match.index + match[0].length
+		});
+		match = frontmatterPattern.exec(content);
+	}
+	for (let i = 0; i < boundaries.length; i++) {
+		const boundary = boundaries[i];
+		const nextStart = i + 1 < boundaries.length ? boundaries[i + 1].start : content.length;
+		const sectionContent = content.slice(boundary.start, nextStart);
+		sections.push({
+			url: boundary.url,
+			raw: sectionContent.trimEnd()
+		});
+	}
+	return sections;
+}
+/**
+* Filter API page content sections from global llms-full.txt.
+*
+* Sections are delimited by `---\nurl: {path}\n---` frontmatter blocks.
+* Removes entire sections whose URL matches a known API route.
+*
+* @param content - Full llms-full.txt content string
+* @param apiRoutes - Set of API route paths to remove
+* @returns Filtered llms-full.txt content
+*/
+function filterLlmsFullTxt(content, apiRoutes) {
+	if (content.trim() === "") return "";
+	const kept = parseSections(content).filter((section) => !apiRoutes.has(section.url));
+	if (kept.length === 0) return "";
+	return kept.map((section) => section.raw).join("\n\n\n");
+}
+/**
+* Format a single llms.txt link entry.
+*/
+function formatEntry(entry) {
+	if (entry.description) return `- [${entry.title}](${entry.url}): ${entry.description}`;
+	return `- [${entry.title}](${entry.url})`;
+}
+/**
+* Generate a per-package llms.txt index.
+*
+* Output format:
+* ```
+* # {name}
+*
+* ## Guides
+*
+* - [Guide Title](/path): Description
+*
+* ## API Reference
+*
+* - [ApiItem](/path): Description
+* ```
+*
+* Sections with no entries are omitted.
+*
+* @param input - Package name, guide pages, and API pages
+* @returns Generated llms.txt content
+*/
+function generatePackageLlmsTxt(input) {
+	const parts = [
+		`# ${input.name}`,
+		"",
+		`> API documentation for the ${input.packageName} package`
+	];
+	if (input.guidePages.length > 0) {
+		parts.push("");
+		parts.push("## Guides");
+		parts.push("");
+		for (const page of input.guidePages) parts.push(formatEntry(page));
+	}
+	if (input.apiPages.length > 0) {
+		parts.push("");
+		parts.push("## API Reference");
+		parts.push("");
+		for (const page of input.apiPages) parts.push(formatEntry(page));
+	}
+	parts.push("");
+	return parts.join("\n");
+}
+/**
+* Concatenate page contents with frontmatter delimiters.
+*
+* Used for llms-full.txt, llms-docs.txt, and llms-api.txt generation
+* (pass different page sets for each).
+*
+* Output format:
+* ```
+* ---
+* url: /path/to/page
+* ---
+*
+* Content here...
+*
+*
+* ---
+* url: /path/to/next
+* ---
+*
+* More content...
+* ```
+*
+* @param pages - Array of page URLs and their markdown content
+* @returns Concatenated content with frontmatter delimiters
+*/
+function generatePackageLlmsFullTxt(pages) {
+	if (pages.length === 0) return "";
+	const sections = [];
+	for (const page of pages) sections.push(`---\nurl: ${page.url}\n---\n\n${page.content}`);
+	return sections.join("\n\n\n");
+}
+
+//#endregion
+export { filterLlmsFullTxt, filterLlmsTxt, generatePackageLlmsFullTxt, generatePackageLlmsTxt, generateStructuredLlmsTxt, parseLlmsTxtLine };

package/llms-program.js

@@ -0,0 +1,262 @@
+import { filterLlmsFullTxt, filterLlmsTxt, generatePackageLlmsFullTxt, generatePackageLlmsTxt, generateStructuredLlmsTxt, parseLlmsTxtLine } from "./llms-processing.js";
+import path from "node:path";
+import { Effect } from "effect";
+import { FileSystem } from "@effect/platform";
+
+//#region src/llms-program.ts
+/**
+* Effect program for post-processing LLMs text files in afterBuild.
+*
+* Wires the pure processing functions from llms-processing.ts into the
+* plugin lifecycle, handling file I/O via \@effect/platform FileSystem.
+*
+* Responsibilities:
+* 1. Collect all API routes from build results
+* 2. Post-process global llms.txt (filter API entries, append pointers)
+* 3. Post-process global llms-full.txt (remove API sections)
+* 4. Generate per-package files (llms.txt, llms-full.txt, llms-docs.txt, llms-api.txt)
+*/
+/**
+* Convert generated file relative paths to route URLs for matching against llms.txt entries.
+*
+* The generatedFiles Set stores paths relative to the output dir, like "class/pipeline.mdx".
+* The llms.txt entries have URLs like "/api/class/pipeline", so we:
+* - Replace .mdx extension with .md (RSPress llms.txt URLs use .md extension)
+* - Prepend the baseRoute (e.g., "/api/")
+* - Normalize trailing slashes
+*/
+function buildApiRoutes(buildResults) {
+	const apiRoutes = /* @__PURE__ */ new Set();
+	for (const result of buildResults) {
+		const base = result.baseRoute.endsWith("/") ? result.baseRoute : `${result.baseRoute}/`;
+		for (const relPath of result.generatedFiles) {
+			if (!relPath.endsWith(".mdx")) continue;
+			const routeUrl = `${base}${relPath.replace(/\.mdx$/, ".md")}`;
+			apiRoutes.add(routeUrl);
+		}
+	}
+	return apiRoutes;
+}
+/**
+* Discover version/locale prefixes from build results' base routes.
+*
+* RSPress generates global llms files at version/locale prefixed paths:
+* - dist/llms.txt (default)
+* - dist/v1/llms.txt (versioned)
+* - dist/zh/llms.txt (i18n)
+*
+* We examine the base routes to extract prefixes. Base routes like:
+* - "/api" results in prefix ""
+* - "/v1/api" results in prefix "v1"
+* - "/zh/api" results in prefix "zh"
+*
+* Returns unique prefixes including "" (root) which always exists.
+*/
+function discoverPrefixes(buildResults) {
+	const prefixes = /* @__PURE__ */ new Set();
+	prefixes.add("");
+	for (const result of buildResults) {
+		const segments = result.baseRoute.split("/").filter(Boolean);
+		if (segments.length > 1) {
+			const prefixSegments = segments.slice(0, -1);
+			prefixes.add(prefixSegments.join("/"));
+		}
+	}
+	return prefixes;
+}
+/**
+* Build package pointers for a given prefix context.
+*
+* Each build result produces a per-package llms.txt file. The pointer
+* URL is relative to the site root and includes any prefix.
+*/
+function buildPackagePointers(buildResults, prefix, packageRoutes) {
+	const pointers = [];
+	for (const result of buildResults) {
+		if (prefix !== "" && !result.baseRoute.startsWith(`/${prefix}/`)) continue;
+		const displayName = result.apiName ?? result.packageName;
+		const pkgRoute = packageRoutes.get(result.packageName) ?? result.baseRoute;
+		const base = pkgRoute.endsWith("/") ? pkgRoute : `${pkgRoute}/`;
+		pointers.push({
+			name: displayName,
+			llmsTxtUrl: `${base}llms.txt`
+		});
+	}
+	return pointers;
+}
+/**
+* Collect API page entries from the global llms.txt for a specific package.
+*
+* Parses the global llms.txt to find entries whose URLs match this package's
+* generated API routes, building the LlmsTxtEntry array for per-package files.
+*/
+function collectApiEntries(globalLlmsTxtContent, result) {
+	const base = result.baseRoute.endsWith("/") ? result.baseRoute : `${result.baseRoute}/`;
+	const entries = [];
+	for (const line of globalLlmsTxtContent.split("\n")) {
+		const entry = parseLlmsTxtLine(line);
+		if (!entry) continue;
+		if (entry.url.startsWith(base)) entries.push(entry);
+	}
+	return entries;
+}
+/**
+* Collect guide page entries from the global llms.txt that are NOT API pages.
+*
+* Guide pages are entries in the global llms.txt that are under the package's
+* route but not in the API routes set. Uses packageRoute (e.g., "/kitchensink")
+* instead of prefix to avoid matching entries from other packages.
+*/
+function collectGuideEntries(globalLlmsTxtContent, apiRoutes, packageRoute) {
+	const base = packageRoute.endsWith("/") ? packageRoute : `${packageRoute}/`;
+	const entries = [];
+	for (const line of globalLlmsTxtContent.split("\n")) {
+		const entry = parseLlmsTxtLine(line);
+		if (!entry) continue;
+		if ((entry.url === packageRoute || entry.url.startsWith(base)) && !apiRoutes.has(entry.url)) entries.push(entry);
+	}
+	return entries;
+}
+/**
+* Extract page content sections from llms-full.txt that match a URL predicate.
+*
+* Sections are delimited by `---\nurl: {path}\n---` frontmatter blocks.
+* Returns content (without frontmatter) for sections whose URL satisfies the predicate.
+*/
+function extractSections(globalLlmsFullContent, urlPredicate) {
+	const pages = [];
+	if (!globalLlmsFullContent) return pages;
+	const frontmatterPattern = /^---\nurl:\s*(.+)\n---$/gm;
+	let match = frontmatterPattern.exec(globalLlmsFullContent);
+	const boundaries = [];
+	while (match !== null) {
+		boundaries.push({
+			url: match[1].trim(),
+			start: match.index,
+			fmEnd: match.index + match[0].length
+		});
+		match = frontmatterPattern.exec(globalLlmsFullContent);
+	}
+	for (let i = 0; i < boundaries.length; i++) {
+		const boundary = boundaries[i];
+		if (!urlPredicate(boundary.url)) continue;
+		const nextStart = i + 1 < boundaries.length ? boundaries[i + 1].start : globalLlmsFullContent.length;
+		const content = globalLlmsFullContent.slice(boundary.fmEnd, nextStart).trim();
+		pages.push({
+			url: boundary.url,
+			content
+		});
+	}
+	return pages;
+}
+/**
+* Read llms-full.txt sections and collect page content for API pages.
+*
+* Parses the global llms-full.txt to extract full markdown content for
+* pages matching this package's API routes.
+*/
+function collectApiPageContent(globalLlmsFullContent, result) {
+	const base = result.baseRoute.endsWith("/") ? result.baseRoute : `${result.baseRoute}/`;
+	return extractSections(globalLlmsFullContent, (url) => url.startsWith(base));
+}
+/**
+* Process LLMs text files after the RSPress build.
+*
+* This Effect program:
+* 1. Builds the set of all API route URLs from build results
+* 2. Discovers version/locale prefixes from base routes
+* 3. For each prefix, post-processes global llms.txt and llms-full.txt
+* 4. When scopes is enabled, generates per-package LLMs files
+*
+* Requires FileSystem from \@effect/platform.
+*/
+function processLlmsFiles(input) {
+	return Effect.gen(function* () {
+		const fs = yield* FileSystem.FileSystem;
+		const { outDir, buildResults, llmsPlugin, packageRoutes } = input;
+		if (buildResults.length === 0) return;
+		const apiRoutes = buildApiRoutes(buildResults);
+		yield* Effect.logDebug(`Built ${apiRoutes.size} API routes for LLMs filtering`);
+		if (apiRoutes.size === 0) return;
+		const prefixes = discoverPrefixes(buildResults);
+		yield* Effect.forEach([...prefixes], (prefix) => processPrefix(fs, outDir, prefix, buildResults, apiRoutes, llmsPlugin, packageRoutes), { concurrency: "unbounded" });
+	});
+}
+/**
+* Process global and per-package LLMs files for a single prefix.
+*/
+function processPrefix(fs, outDir, prefix, buildResults, apiRoutes, llmsPlugin, packageRoutes) {
+	return Effect.gen(function* () {
+		const prefixDir = prefix ? path.join(outDir, prefix) : outDir;
+		const llmsTxtPath = path.join(prefixDir, "llms.txt");
+		if (!(yield* fs.exists(llmsTxtPath).pipe(Effect.orElseSucceed(() => false)))) return;
+		const llmsTxtContent = yield* fs.readFileString(llmsTxtPath).pipe(Effect.orDie);
+		const llmsFullTxtPath = path.join(prefixDir, "llms-full.txt");
+		const llmsFullTxtContent = (yield* fs.exists(llmsFullTxtPath).pipe(Effect.orElseSucceed(() => false))) ? yield* fs.readFileString(llmsFullTxtPath).pipe(Effect.orDie) : "";
+		if (llmsPlugin.scopes) {
+			const structuredLlmsTxt = generateStructuredLlmsTxt(llmsTxtContent, apiRoutes, buildResults.map((r) => {
+				const pkgRoute = packageRoutes.get(r.packageName) ?? r.baseRoute;
+				return {
+					name: r.apiName ?? r.packageName,
+					packageName: r.packageName,
+					version: r.packageVersion,
+					description: r.packageDescription,
+					packageRoute: pkgRoute,
+					llmsApiTxtUrl: `${pkgRoute.endsWith("/") ? pkgRoute : `${pkgRoute}/`}llms-api.txt`
+				};
+			}));
+			yield* fs.writeFileString(llmsTxtPath, structuredLlmsTxt).pipe(Effect.orDie);
+		} else {
+			const filteredLlmsTxt = filterLlmsTxt(llmsTxtContent, apiRoutes, buildPackagePointers(buildResults, prefix, packageRoutes));
+			yield* fs.writeFileString(llmsTxtPath, filteredLlmsTxt).pipe(Effect.orDie);
+		}
+		if (llmsFullTxtContent) {
+			const filteredLlmsFullTxt = filterLlmsFullTxt(llmsFullTxtContent, apiRoutes);
+			yield* fs.writeFileString(llmsFullTxtPath, filteredLlmsFullTxt).pipe(Effect.orDie);
+		}
+		if (llmsPlugin.scopes) {
+			const prefixResults = prefix === "" ? [...buildResults] : buildResults.filter((r) => r.baseRoute.startsWith(`/${prefix}/`));
+			yield* Effect.forEach(prefixResults, (result) => generatePerPackageFiles(fs, outDir, result, llmsTxtContent, llmsFullTxtContent, apiRoutes, llmsPlugin, packageRoutes.get(result.packageName) ?? result.baseRoute), { concurrency: "unbounded" });
+		}
+	});
+}
+/**
+* Generate per-package LLMs files (llms.txt, llms-full.txt, llms-docs.txt, llms-api.txt).
+*/
+function generatePerPackageFiles(fs, outDir, result, globalLlmsTxtContent, globalLlmsFullContent, apiRoutes, llmsPlugin, packageRoute) {
+	return Effect.gen(function* () {
+		const pkgRouteSegment = packageRoute.replace(/^\//, "");
+		const packageLlmsDir = pkgRouteSegment ? path.join(outDir, pkgRouteSegment) : outDir;
+		yield* fs.makeDirectory(packageLlmsDir, { recursive: true }).pipe(Effect.orDie);
+		const displayName = result.apiName ?? result.packageName;
+		const apiEntries = collectApiEntries(globalLlmsTxtContent, result);
+		const guideEntries = collectGuideEntries(globalLlmsTxtContent, apiRoutes, packageRoute);
+		const packageLlmsTxt = generatePackageLlmsTxt({
+			name: displayName,
+			packageName: result.packageName,
+			guidePages: guideEntries,
+			apiPages: apiEntries
+		});
+		yield* fs.writeFileString(path.join(packageLlmsDir, "llms.txt"), packageLlmsTxt).pipe(Effect.orDie);
+		const apiPageContent = collectApiPageContent(globalLlmsFullContent, result);
+		const guideRouteUrls = new Set(guideEntries.map((e) => e.url));
+		const guidePageContent = globalLlmsFullContent ? extractSections(globalLlmsFullContent, (url) => guideRouteUrls.has(url)) : [];
+		const fullPageContent = [...guidePageContent, ...apiPageContent];
+		if (fullPageContent.length > 0) {
+			const packageLlmsFullTxt = generatePackageLlmsFullTxt(fullPageContent);
+			yield* fs.writeFileString(path.join(packageLlmsDir, "llms-full.txt"), packageLlmsFullTxt).pipe(Effect.orDie);
+		}
+		if (llmsPlugin.apiTxt && apiPageContent.length > 0) {
+			const apiTxtContent = generatePackageLlmsFullTxt(apiPageContent);
+			yield* fs.writeFileString(path.join(packageLlmsDir, "llms-api.txt"), apiTxtContent).pipe(Effect.orDie);
+		}
+		if (guidePageContent.length > 0) {
+			const docsTxtContent = generatePackageLlmsFullTxt(guidePageContent);
+			yield* fs.writeFileString(path.join(packageLlmsDir, "llms-docs.txt"), docsTxtContent).pipe(Effect.orDie);
+		}
+		yield* Effect.logDebug(`Generated LLMs files for ${displayName} in ${packageLlmsDir}`);
+	});
+}
+
+//#endregion
+export { processLlmsFiles };