struth 1.0.0

package/src/core/pipeline/discover.ts

@@ -0,0 +1,448 @@
+import type { z } from "zod";
+import { USER_AGENT } from "../constants.js";
+import { DiscoverResult, type MirrorOptions } from "../schemas.js";
+
+type DiscoverOpts = Pick<
+	z.infer<typeof MirrorOptions>,
+	"smart" | "filter" | "excludePath" | "exclude" | "top"
+>;
+
+// ── Non-doc file extensions to filter ───────────────────────────────
+
+const NON_DOC_EXTENSIONS = new Set([
+	".json",
+	".xml",
+	".txt",
+	".png",
+	".jpg",
+	".svg",
+	".css",
+	".js",
+	".ico",
+	".gif",
+	".woff",
+	".woff2",
+	".ttf",
+	".eot",
+	".pdf",
+]);
+
+// ── Platform detection patterns ─────────────────────────────────────
+
+const PLATFORM_PATTERNS: Array<{
+	name: string;
+	test: (html: string, headers: Headers) => boolean;
+}> = [
+	{
+		name: "ReadTheDocs",
+		test: (html) =>
+			html.includes("readthedocs") || html.includes("Read the Docs") || html.includes("rtd.css"),
+	},
+	{
+		name: "GitBook",
+		test: (html, headers) =>
+			html.includes("gitbook") || headers.get("x-served-by")?.includes("gitbook") === true,
+	},
+	{
+		name: "Docusaurus",
+		test: (html) => html.includes("docusaurus") || html.includes("__docusaurus"),
+	},
+	{
+		name: "MkDocs",
+		test: (html) => html.includes("mkdocs") || html.includes("MkDocs"),
+	},
+	{
+		name: "Sphinx",
+		test: (html) =>
+			html.includes("sphinx") || html.includes("Sphinx") || html.includes("_sphinx_javascript"),
+	},
+	{
+		name: "Mintlify",
+		test: (html) => html.includes("mintlify"),
+	},
+	{
+		name: "Nextra",
+		test: (html) => html.includes("nextra"),
+	},
+	{
+		name: "VitePress",
+		test: (html) => html.includes("vitepress") || html.includes("VitePress"),
+	},
+	{
+		name: "mdBook",
+		test: (html) => html.includes("mdbook") || html.includes("mdBook"),
+	},
+];
+
+// ── Timeouts ────────────────────────────────────────────────────────
+
+const PROBE_TIMEOUT = 5_000;
+const WALK_TIMEOUT = 10_000;
+
+// ── Fetch helpers ───────────────────────────────────────────────────
+
+async function probe(url: string, timeout = PROBE_TIMEOUT): Promise<Response | null> {
+	try {
+		const controller = new AbortController();
+		const timer = setTimeout(() => controller.abort(), timeout);
+		const resp = await fetch(url, {
+			headers: { "User-Agent": USER_AGENT },
+			signal: controller.signal,
+			redirect: "follow",
+		});
+		clearTimeout(timer);
+		return resp.ok ? resp : null;
+	} catch {
+		return null;
+	}
+}
+
+async function fetchText(url: string, timeout = WALK_TIMEOUT): Promise<string | null> {
+	try {
+		const controller = new AbortController();
+		const timer = setTimeout(() => controller.abort(), timeout);
+		const resp = await fetch(url, {
+			headers: { "User-Agent": USER_AGENT },
+			signal: controller.signal,
+			redirect: "follow",
+		});
+		clearTimeout(timer);
+		if (!resp.ok) return null;
+		return await resp.text();
+	} catch {
+		return null;
+	}
+}
+
+// ── URL processing ──────────────────────────────────────────────────
+
+function normalizeUrl(raw: string): string {
+	try {
+		const parsed = new URL(raw);
+		parsed.hash = ""; // strip fragments
+		// normalize trailing slash: remove if path has content beyond /
+		if (parsed.pathname.length > 1 && parsed.pathname.endsWith("/")) {
+			parsed.pathname = parsed.pathname.slice(0, -1);
+		}
+		return parsed.href;
+	} catch {
+		return raw;
+	}
+}
+
+function isDocUrl(urlStr: string): boolean {
+	try {
+		const parsed = new URL(urlStr);
+		const ext = parsed.pathname.match(/\.[a-z0-9]+$/i)?.[0]?.toLowerCase();
+		if (ext && NON_DOC_EXTENSIONS.has(ext)) return false;
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+function isSameOrigin(urlStr: string, origin: string): boolean {
+	try {
+		const parsed = new URL(urlStr);
+		return parsed.origin === origin;
+	} catch {
+		return false;
+	}
+}
+
+type UrlEntry = { url: string; source: z.infer<typeof DiscoverResult>["urls"][number]["source"] };
+
+function processUrls(
+	raw: UrlEntry[],
+	origin: string,
+	opts: DiscoverOpts,
+): { urls: UrlEntry[]; totalFound: number; afterDedup: number } {
+	const totalFound = raw.length;
+
+	// Normalize, filter non-doc, filter external
+	let urls = raw
+		.map((e) => ({ ...e, url: normalizeUrl(e.url) }))
+		.filter((e) => isDocUrl(e.url))
+		.filter((e) => isSameOrigin(e.url, origin));
+
+	// Deduplicate
+	const seen = new Set<string>();
+	urls = urls.filter((e) => {
+		if (seen.has(e.url)) return false;
+		seen.add(e.url);
+		return true;
+	});
+
+	const afterDedup = urls.length;
+
+	// Apply excludePath patterns
+	if (opts.excludePath && opts.excludePath.length > 0) {
+		urls = urls.filter((e) => {
+			const path = new URL(e.url).pathname;
+			return !opts.excludePath.some((pat) => path.includes(pat));
+		});
+	}
+
+	// Apply filter (keep ONLY matching)
+	if (opts.filter) {
+		const filterStr = opts.filter;
+		urls = urls.filter((e) => e.url.includes(filterStr));
+	}
+
+	// Apply top limit
+	if (opts.top && opts.top > 0) {
+		urls = urls.slice(0, opts.top);
+	}
+
+	return { urls, totalFound, afterDedup };
+}
+
+// ── Robots.txt check ────────────────────────────────────────────────
+
+async function checkRobots(origin: string): Promise<"allowed" | "blocked" | "no_robots_txt"> {
+	const text = await fetchText(`${origin}/robots.txt`, PROBE_TIMEOUT);
+	if (text === null) return "no_robots_txt";
+
+	// Parse robots.txt for Struth-Bot rules
+	const lines = text.split("\n").map((l) => l.trim());
+	let inStruthBlock = false;
+
+	for (const line of lines) {
+		const lower = line.toLowerCase();
+		if (lower.startsWith("user-agent:")) {
+			const agent = lower.slice("user-agent:".length).trim();
+			inStruthBlock = agent === "struth-bot" || agent === "struth-bot/0.1";
+		} else if (inStruthBlock && lower.startsWith("disallow:")) {
+			const path = lower.slice("disallow:".length).trim();
+			if (path === "/" || path === "/*") return "blocked";
+		}
+	}
+
+	return "allowed";
+}
+
+// ── Platform detection ──────────────────────────────────────────────
+
+function detectPlatform(html: string, headers: Headers): string | null {
+	for (const { name, test } of PLATFORM_PATTERNS) {
+		if (test(html, headers)) return name;
+	}
+	return null;
+}
+
+// ── Discovery strategies ────────────────────────────────────────────
+
+function extractUrlsFromText(text: string, origin: string): string[] {
+	// Extract URLs that look like they belong to the same docs site
+	const urlRegex = /https?:\/\/[^\s<>"')\]]+/g;
+	const matches = text.match(urlRegex) || [];
+	return matches.filter((u) => {
+		try {
+			return new URL(u).origin === origin;
+		} catch {
+			return false;
+		}
+	});
+}
+
+async function tryLlmsFullTxt(origin: string): Promise<UrlEntry[] | null> {
+	const resp = await probe(`${origin}/llms-full.txt`);
+	if (!resp) return null;
+	const text = await resp.text();
+	const urls = extractUrlsFromText(text, origin);
+	if (urls.length === 0) return null;
+	return urls.map((url) => ({ url, source: "llms_full_txt" as const }));
+}
+
+async function tryLlmsTxt(origin: string): Promise<UrlEntry[] | null> {
+	const resp = await probe(`${origin}/llms.txt`);
+	if (!resp) return null;
+	const text = await resp.text();
+	const urls = extractUrlsFromText(text, origin);
+	if (urls.length === 0) return null;
+	return urls.map((url) => ({ url, source: "llms_txt" as const }));
+}
+
+async function tryMdSuffix(url: string): Promise<UrlEntry[] | null> {
+	const mdUrl = url.endsWith("/") ? `${url.slice(0, -1)}.md` : `${url}.md`;
+	const resp = await probe(mdUrl);
+	if (!resp) return null;
+	return [{ url: mdUrl, source: "md_suffix" as const }];
+}
+
+async function trySitemap(origin: string): Promise<UrlEntry[] | null> {
+	const text = await fetchText(`${origin}/sitemap.xml`, WALK_TIMEOUT);
+	if (!text) return null;
+
+	// Parse <loc> elements from XML
+	const locRegex = /<loc>\s*(.*?)\s*<\/loc>/g;
+	const urls: string[] = [];
+	let match: RegExpExecArray | null;
+	match = locRegex.exec(text);
+	while (match !== null) {
+		urls.push(match[1]);
+		match = locRegex.exec(text);
+	}
+
+	if (urls.length === 0) return null;
+	return urls.map((url) => ({ url, source: "sitemap" as const }));
+}
+
+async function tryFirecrawl(url: string, top: number): Promise<UrlEntry[] | null> {
+	const apiKey = process.env.FIRECRAWL_API_KEY;
+	if (!apiKey) return null;
+
+	try {
+		const controller = new AbortController();
+		const timer = setTimeout(() => controller.abort(), WALK_TIMEOUT);
+		const resp = await fetch("https://api.firecrawl.dev/v1/map", {
+			method: "POST",
+			headers: {
+				"Content-Type": "application/json",
+				Authorization: `Bearer ${apiKey}`,
+			},
+			body: JSON.stringify({ url, limit: top }),
+			signal: controller.signal,
+		});
+		clearTimeout(timer);
+
+		if (!resp.ok) return null;
+		const data = (await resp.json()) as { links?: string[] };
+		if (!data.links || data.links.length === 0) return null;
+		return data.links.map((u: string) => ({ url: u, source: "firecrawl" as const }));
+	} catch {
+		return null;
+	}
+}
+
+async function tryLinkWalk(url: string, origin: string): Promise<UrlEntry[] | null> {
+	const text = await fetchText(url, WALK_TIMEOUT);
+	if (!text) return null;
+
+	// Extract href values from anchor tags
+	const hrefRegex = /href=["']([^"']+)["']/g;
+	const urls: string[] = [];
+	let match: RegExpExecArray | null;
+	match = hrefRegex.exec(text);
+	while (match !== null) {
+		const href = match[1];
+		try {
+			// Resolve relative URLs
+			const resolved = new URL(href, url).href;
+			if (new URL(resolved).origin === origin) {
+				urls.push(resolved);
+			}
+		} catch {
+			// skip malformed hrefs (mailto:, javascript:, etc.)
+		}
+		match = hrefRegex.exec(text);
+	}
+
+	if (urls.length === 0) return null;
+	return urls.map((u) => ({ url: u, source: "link_walk" as const }));
+}
+
+// ── Main discover function ──────────────────────────────────────────
+
+/**
+ * Discover documentation pages from a URL.
+ * Waterfall: llms-full.txt -> llms.txt -> .md suffix -> sitemap.xml -> Firecrawl /map -> link walk
+ */
+export async function discover(
+	url: string,
+	opts: DiscoverOpts,
+): Promise<z.infer<typeof DiscoverResult>> {
+	const parsed = new URL(url);
+	const origin = parsed.origin;
+
+	// Check robots.txt in parallel with discovery
+	const robotsPromise = checkRobots(origin);
+
+	// Fetch root page for platform detection (we may need it later for link_walk too)
+	let rootHtml: string | null = null;
+	let rootHeaders: Headers | null = null;
+
+	async function getRootPage(): Promise<{ html: string; headers: Headers } | null> {
+		if (rootHtml !== null && rootHeaders !== null) {
+			return { html: rootHtml, headers: rootHeaders };
+		}
+		try {
+			const controller = new AbortController();
+			const timer = setTimeout(() => controller.abort(), WALK_TIMEOUT);
+			const resp = await fetch(url, {
+				headers: { "User-Agent": USER_AGENT },
+				signal: controller.signal,
+				redirect: "follow",
+			});
+			clearTimeout(timer);
+			if (!resp.ok) return null;
+			rootHtml = await resp.text();
+			rootHeaders = resp.headers;
+			return { html: rootHtml, headers: rootHeaders };
+		} catch {
+			return null;
+		}
+	}
+
+	// Waterfall strategy
+	type SourceMethod = z.infer<typeof DiscoverResult>["urls"][number]["source"];
+	let rawUrls: UrlEntry[] | null = null;
+	let sourceMethod: SourceMethod = "link_walk";
+
+	// 1. llms-full.txt
+	rawUrls = await tryLlmsFullTxt(origin);
+	if (rawUrls) {
+		sourceMethod = "llms_full_txt";
+	}
+
+	// 2. llms.txt
+	if (!rawUrls) {
+		rawUrls = await tryLlmsTxt(origin);
+		if (rawUrls) sourceMethod = "llms_txt";
+	}
+
+	// 3. .md suffix
+	if (!rawUrls) {
+		rawUrls = await tryMdSuffix(url);
+		if (rawUrls) sourceMethod = "md_suffix";
+	}
+
+	// 4. sitemap.xml
+	if (!rawUrls) {
+		rawUrls = await trySitemap(origin);
+		if (rawUrls) sourceMethod = "sitemap";
+	}
+
+	// 5. Firecrawl
+	if (!rawUrls) {
+		rawUrls = await tryFirecrawl(url, opts.top ?? 20);
+		if (rawUrls) sourceMethod = "firecrawl";
+	}
+
+	// 6. link_walk
+	if (!rawUrls) {
+		rawUrls = await tryLinkWalk(url, origin);
+		if (rawUrls) sourceMethod = "link_walk";
+	}
+
+	// Process URLs
+	const { urls, totalFound, afterDedup } = processUrls(rawUrls || [], origin, opts);
+
+	// Platform detection
+	const root = await getRootPage();
+	const platform = root ? detectPlatform(root.html, root.headers) : null;
+
+	// Robots.txt
+	const robotsStatus = await robotsPromise;
+
+	const result = {
+		urls: urls.map((e) => ({ url: e.url, source: e.source })),
+		source_method: sourceMethod,
+		total_found: totalFound,
+		after_dedup: afterDedup,
+		platform_detected: platform,
+		robots_txt_status: robotsStatus,
+	};
+
+	return DiscoverResult.parse(result);
+}

package/src/core/pipeline/integrity.ts

@@ -0,0 +1,214 @@
+import type { z } from "zod";
+import { SCHEMA_VERSION } from "../constants.js";
+import type { ContentIntegrity, StructuralMetrics } from "../schemas.js";
+
+/**
+ * Run the Content Integrity Pipeline on a page's content.
+ * Steps: Unicode NFC normalization, structural anomaly detection,
+ * OWASP LLM01 alignment check, structural isolation.
+ */
+export async function assessIntegrity(
+	content: string,
+	_sourceUrl: string,
+): Promise<z.infer<typeof ContentIntegrity>> {
+	// Unicode NFC normalization
+	const _normalized = content.normalize("NFC");
+
+	// Anomaly detection deferred to Sprint 3
+	const flaggedAnomalies: string[] = [];
+
+	// Structural baseline: 1 - (flagged_anomalies.length * 0.1), clamped [0, 1]
+	const structuralBaseline = Math.max(0, Math.min(1, 1 - flaggedAnomalies.length * 0.1));
+
+	return {
+		unicode_normalized: true,
+		structural_baseline: structuralBaseline,
+		flagged_anomalies: flaggedAnomalies,
+		owasp_llm01_checked: true,
+		pipeline_version: SCHEMA_VERSION,
+	};
+}
+
+/** Imperative verbs that commonly start doc instruction sentences. */
+const IMPERATIVE_VERBS = new Set([
+	"run",
+	"install",
+	"set",
+	"create",
+	"add",
+	"use",
+	"configure",
+	"enable",
+	"click",
+	"open",
+	"copy",
+	"move",
+	"delete",
+	"remove",
+	"update",
+	"check",
+	"verify",
+	"ensure",
+	"start",
+	"stop",
+	"build",
+	"deploy",
+	"test",
+	"import",
+	"export",
+	"define",
+	"specify",
+	"select",
+	"enter",
+	"type",
+	"navigate",
+	"go",
+	"download",
+	"upload",
+	"save",
+	"load",
+	"execute",
+	"apply",
+	"include",
+	"exclude",
+	"pass",
+	"return",
+	"call",
+	"send",
+	"fetch",
+	"get",
+	"put",
+	"post",
+	"patch",
+	"replace",
+	"merge",
+	"wrap",
+	"mount",
+	"bind",
+	"attach",
+	"listen",
+	"emit",
+	"register",
+	"subscribe",
+	"publish",
+	"connect",
+	"disconnect",
+	"initialize",
+	"setup",
+	"reset",
+	"clear",
+	"flush",
+	"close",
+	"shutdown",
+	"restart",
+	"log",
+	"print",
+	"debug",
+	"trace",
+	"monitor",
+	"watch",
+	"observe",
+	"inspect",
+	"try",
+	"catch",
+	"throw",
+	"handle",
+	"retry",
+	"note",
+	"see",
+	"refer",
+	"visit",
+	"read",
+	"write",
+	"append",
+	"prepend",
+	"insert",
+	"override",
+	"extend",
+	"implement",
+	"inherit",
+	"compose",
+]);
+
+/**
+ * Calculate structural metrics for a page's content.
+ */
+export function calculateStructuralMetrics(content: string): z.infer<typeof StructuralMetrics> {
+	return {
+		char_entropy: charEntropy(content),
+		code_block_ratio: codeBlockRatio(content),
+		avg_section_words: avgSectionWords(content),
+		imperative_sentence_ratio: imperativeSentenceRatio(content),
+		total_tokens: Math.ceil(content.length / 4),
+	};
+}
+
+/** Shannon entropy of character distribution. */
+function charEntropy(text: string): number {
+	if (text.length === 0) return 0;
+
+	const freq = new Map<string, number>();
+	for (const ch of text) {
+		freq.set(ch, (freq.get(ch) ?? 0) + 1);
+	}
+
+	let entropy = 0;
+	const len = text.length;
+	for (const count of freq.values()) {
+		const p = count / len;
+		if (p > 0) {
+			entropy -= p * Math.log2(p);
+		}
+	}
+	return entropy;
+}
+
+/** Ratio of characters inside ``` blocks to total characters. */
+function codeBlockRatio(text: string): number {
+	if (text.length === 0) return 0;
+
+	let insideCode = 0;
+	const parts = text.split("```");
+	// Odd-indexed parts are inside code blocks
+	for (let i = 1; i < parts.length; i += 2) {
+		insideCode += parts[i].length;
+	}
+
+	return Math.max(0, Math.min(1, insideCode / text.length));
+}
+
+/** Average words per section (## or higher heading). */
+function avgSectionWords(text: string): number {
+	const words = text.split(/\s+/).filter((w) => w.length > 0);
+	if (words.length === 0) return 0;
+
+	// Count headings: lines starting with # or ## (## or higher = # and ##)
+	const headingCount = text.split("\n").filter((line) => /^#{1,2}\s/.test(line)).length;
+
+	if (headingCount === 0) return words.length;
+	return words.length / headingCount;
+}
+
+/** Fraction of sentences starting with an imperative verb. */
+function imperativeSentenceRatio(text: string): number {
+	// Split on sentence boundaries: period, exclamation, question mark followed by space or end
+	const sentences = text
+		.split(/[.!?](?:\s|$)/)
+		.map((s) => s.trim())
+		.filter((s) => s.length > 0);
+
+	if (sentences.length === 0) return 0;
+
+	let imperativeCount = 0;
+	for (const sentence of sentences) {
+		const firstWord = sentence
+			.split(/\s+/)[0]
+			?.toLowerCase()
+			.replace(/[^a-z]/g, "");
+		if (firstWord && IMPERATIVE_VERBS.has(firstWord)) {
+			imperativeCount++;
+		}
+	}
+
+	return Math.max(0, Math.min(1, imperativeCount / sentences.length));
+}