ruflo 3.5.2 → 3.5.4

package/src/ruvocal/src/lib/utils/marked.ts

@@ -0,0 +1,531 @@
+import katex from "katex";
+import "katex/dist/contrib/mhchem.mjs";
+import { Marked } from "marked";
+import type { Tokens, TokenizerExtension, RendererExtension } from "marked";
+import { parseDocument } from "htmlparser2";
+// Simple type to replace removed WebSearchSource
+type SimpleSource = {
+	title?: string;
+	link: string;
+};
+import hljs from "highlight.js/lib/core";
+import type { LanguageFn } from "highlight.js";
+import javascript from "highlight.js/lib/languages/javascript";
+import typescript from "highlight.js/lib/languages/typescript";
+import json from "highlight.js/lib/languages/json";
+import bash from "highlight.js/lib/languages/bash";
+import shell from "highlight.js/lib/languages/shell";
+import python from "highlight.js/lib/languages/python";
+import go from "highlight.js/lib/languages/go";
+import rust from "highlight.js/lib/languages/rust";
+import java from "highlight.js/lib/languages/java";
+import csharp from "highlight.js/lib/languages/csharp";
+import cpp from "highlight.js/lib/languages/cpp";
+import cLang from "highlight.js/lib/languages/c";
+import xml from "highlight.js/lib/languages/xml";
+import css from "highlight.js/lib/languages/css";
+import scss from "highlight.js/lib/languages/scss";
+import markdownLang from "highlight.js/lib/languages/markdown";
+import yaml from "highlight.js/lib/languages/yaml";
+import sql from "highlight.js/lib/languages/sql";
+import plaintext from "highlight.js/lib/languages/plaintext";
+import { parseIncompleteMarkdown } from "./parseIncompleteMarkdown";
+import { parseMarkdownIntoBlocks } from "./parseBlocks";
+
+const bundledLanguages: [string, LanguageFn][] = [
+	["javascript", javascript],
+	["typescript", typescript],
+	["json", json],
+	["bash", bash],
+	["shell", shell],
+	["python", python],
+	["go", go],
+	["rust", rust],
+	["java", java],
+	["csharp", csharp],
+	["cpp", cpp],
+	["c", cLang],
+	["xml", xml],
+	["html", xml],
+	["css", css],
+	["scss", scss],
+	["markdown", markdownLang],
+	["yaml", yaml],
+	["sql", sql],
+	["plaintext", plaintext],
+];
+
+bundledLanguages.forEach(([name, language]) => hljs.registerLanguage(name, language));
+
+// Media URL detection
+const VIDEO_EXTENSIONS = /\.(mp4|webm|ogg|mov|m4v)([?#]|$)/i;
+const AUDIO_EXTENSIONS = /\.(mp3|wav|m4a|aac|flac)([?#]|$)/i;
+
+function isVideoUrl(url: string): boolean {
+	return VIDEO_EXTENSIONS.test(url);
+}
+
+function isAudioUrl(url: string): boolean {
+	return AUDIO_EXTENSIONS.test(url);
+}
+
+// Multimedia HTML sanitization (works in Web Workers - no DOM needed)
+const MULTIMEDIA_TAGS = new Set(["video", "source", "audio"]);
+const MULTIMEDIA_ALLOWED_ATTRS = new Set([
+	"src",
+	"type",
+	"controls",
+	"autoplay",
+	"loop",
+	"muted",
+	"playsinline",
+	"poster",
+	"width",
+	"height",
+	"preload",
+]);
+const MULTIMEDIA_BOOLEAN_ATTRS = new Set(["controls", "autoplay", "loop", "muted", "playsinline"]);
+const MULTIMEDIA_URI_ATTRS = new Set(["src", "poster"]);
+const MULTIMEDIA_ALLOWED_URI_PATTERN = /^(?!javascript:|data:text\/html)/i;
+const MULTIMEDIA_HTML_REGEX = /<\/?(video|source|audio)\b/i;
+
+type HtmlNode = {
+	type: string;
+	name?: string;
+	attribs?: Record<string, string>;
+	children?: HtmlNode[];
+	data?: string;
+};
+
+interface katexBlockToken extends Tokens.Generic {
+	type: "katexBlock";
+	raw: string;
+	text: string;
+	displayMode: true;
+}
+
+interface katexInlineToken extends Tokens.Generic {
+	type: "katexInline";
+	raw: string;
+	text: string;
+	displayMode: false;
+}
+
+export const katexBlockExtension: TokenizerExtension & RendererExtension = {
+	name: "katexBlock",
+	level: "block",
+
+	start(src: string): number | undefined {
+		const match = src.match(/(\${2}|\\\[)/);
+		return match ? match.index : -1;
+	},
+
+	tokenizer(src: string): katexBlockToken | undefined {
+		// 1) $$ ... $$
+		const rule1 = /^\${2}([\s\S]+?)\${2}/;
+		const match1 = rule1.exec(src);
+		if (match1) {
+			const token: katexBlockToken = {
+				type: "katexBlock",
+				raw: match1[0],
+				text: match1[1].trim(),
+				displayMode: true,
+			};
+			return token;
+		}
+
+		// 2) \[ ... \]
+		const rule2 = /^\\\[([\s\S]+?)\\\]/;
+		const match2 = rule2.exec(src);
+		if (match2) {
+			const token: katexBlockToken = {
+				type: "katexBlock",
+				raw: match2[0],
+				text: match2[1].trim(),
+				displayMode: true,
+			};
+			return token;
+		}
+
+		return undefined;
+	},
+
+	renderer(token) {
+		if (token.type === "katexBlock") {
+			return katex.renderToString(token.text, {
+				throwOnError: false,
+				displayMode: token.displayMode,
+			});
+		}
+		return undefined;
+	},
+};
+
+const katexInlineExtension: TokenizerExtension & RendererExtension = {
+	name: "katexInline",
+	level: "inline",
+
+	start(src: string): number | undefined {
+		const match = src.match(/(\$|\\\()/);
+		return match ? match.index : -1;
+	},
+
+	tokenizer(src: string): katexInlineToken | undefined {
+		// 1) $...$
+		const rule1 = /^\$([^$]+?)\$/;
+		const match1 = rule1.exec(src);
+		if (match1) {
+			const token: katexInlineToken = {
+				type: "katexInline",
+				raw: match1[0],
+				text: match1[1].trim(),
+				displayMode: false,
+			};
+			return token;
+		}
+
+		// 2) \(...\)
+		const rule2 = /^\\\(([\s\S]+?)\\\)/;
+		const match2 = rule2.exec(src);
+		if (match2) {
+			const token: katexInlineToken = {
+				type: "katexInline",
+				raw: match2[0],
+				text: match2[1].trim(),
+				displayMode: false,
+			};
+			return token;
+		}
+
+		return undefined;
+	},
+
+	renderer(token) {
+		if (token.type === "katexInline") {
+			return katex.renderToString(token.text, {
+				throwOnError: false,
+				displayMode: token.displayMode,
+			});
+		}
+		return undefined;
+	},
+};
+
+function escapeHTML(content: string) {
+	return content.replace(
+		/[<>&"']/g,
+		(x) =>
+			({
+				"<": "&lt;",
+				">": "&gt;",
+				"&": "&amp;",
+				"'": "&#39;",
+				'"': "&quot;",
+			})[x] || x
+	);
+}
+
+function addInlineCitations(md: string, webSearchSources: SimpleSource[] = []): string {
+	const linkStyle =
+		"color: rgb(59, 130, 246); text-decoration: none; hover:text-decoration: underline;";
+	return md.replace(/\[(\d+)\]/g, (match: string) => {
+		const indices: number[] = (match.match(/\d+/g) || []).map(Number);
+		const links: string = indices
+			.map((index: number) => {
+				if (index === 0) return false;
+				const source = webSearchSources[index - 1];
+				if (source) {
+					return `<a href="${escapeHTML(source.link)}" target="_blank" rel="noreferrer" style="${linkStyle}">${index}</a>`;
+				}
+				return "";
+			})
+			.filter(Boolean)
+			.join(", ");
+		return links ? ` <sup>${links}</sup>` : match;
+	});
+}
+
+function sanitizeHref(href?: string | null): string | undefined {
+	if (!href) return undefined;
+	const trimmed = href.trim();
+	const lower = trimmed.toLowerCase();
+	if (lower.startsWith("javascript:") || lower.startsWith("data:text/html")) {
+		return undefined;
+	}
+	return trimmed.replace(/>$/, "");
+}
+
+function highlightCode(text: string, lang?: string): string {
+	if (lang && hljs.getLanguage(lang)) {
+		try {
+			return hljs.highlight(text, { language: lang, ignoreIllegals: true }).value;
+		} catch {
+			// fall through to auto-detect
+		}
+	}
+	return hljs.highlightAuto(text).value;
+}
+
+function sanitizeMediaUrl(value: string): string | undefined {
+	const trimmed = value.trim().replace(/>$/, "");
+	if (!MULTIMEDIA_ALLOWED_URI_PATTERN.test(trimmed)) return undefined;
+	return trimmed;
+}
+
+function serializeMediaAttributes(attribs?: Record<string, string>): string {
+	if (!attribs) return "";
+	const parts: string[] = [];
+	for (const [rawName, rawValue] of Object.entries(attribs)) {
+		const name = rawName.toLowerCase();
+		if (!MULTIMEDIA_ALLOWED_ATTRS.has(name)) continue;
+		if (MULTIMEDIA_BOOLEAN_ATTRS.has(name)) {
+			parts.push(name);
+			continue;
+		}
+		let value = rawValue ?? "";
+		if (MULTIMEDIA_URI_ATTRS.has(name)) {
+			const safeUrl = sanitizeMediaUrl(value);
+			if (!safeUrl) continue;
+			value = safeUrl;
+		}
+		parts.push(`${name}="${escapeHTML(value)}"`);
+	}
+	return parts.length ? ` ${parts.join(" ")}` : "";
+}
+
+function serializeMediaNode(node: HtmlNode, state: { hasDisallowedTag: boolean }): string {
+	if (node.type === "text") {
+		return escapeHTML(node.data ?? "");
+	}
+	if (node.type === "tag" || node.type === "script" || node.type === "style") {
+		const tagName = node.name?.toLowerCase() ?? "";
+		if (!MULTIMEDIA_TAGS.has(tagName)) {
+			state.hasDisallowedTag = true;
+			return "";
+		}
+		const attrs = serializeMediaAttributes(node.attribs);
+		if (tagName === "source") {
+			return `<source${attrs}>`;
+		}
+		const children = (node.children ?? [])
+			.map((child) => serializeMediaNode(child, state))
+			.join("");
+		return `<${tagName}${attrs}>${children}</${tagName}>`;
+	}
+	if (node.type === "comment") {
+		return "";
+	}
+	return "";
+}
+
+/**
+ * Sanitizes HTML to allow only video/audio/source tags with safe attributes.
+ * Uses htmlparser2 which works in Web Workers (no DOM needed).
+ * If any disallowed tags are found, escapes the entire input.
+ */
+function sanitizeHtmlForMultimedia(html: string): string {
+	if (!MULTIMEDIA_HTML_REGEX.test(html)) {
+		return escapeHTML(html);
+	}
+	const document = parseDocument(html, {
+		lowerCaseAttributeNames: true,
+		lowerCaseTags: true,
+		recognizeSelfClosing: true,
+	}) as unknown as { children: HtmlNode[] };
+	const state = { hasDisallowedTag: false };
+	const sanitized = (document.children ?? [])
+		.map((child) => serializeMediaNode(child, state))
+		.join("");
+	if (state.hasDisallowedTag) {
+		return escapeHTML(html);
+	}
+	return sanitized;
+}
+
+function createMarkedInstance(sources: SimpleSource[]): Marked {
+	return new Marked({
+		hooks: {
+			postprocess: (html) => addInlineCitations(html, sources),
+		},
+		extensions: [katexBlockExtension, katexInlineExtension],
+		renderer: {
+			link: (href, title, text) => {
+				const safeHref = sanitizeHref(href);
+				return safeHref
+					? `<a href="${escapeHTML(safeHref)}" target="_blank" rel="noreferrer">${text}</a>`
+					: `<span>${escapeHTML(text ?? "")}</span>`;
+			},
+			image: (href, title, text) => {
+				const safeHref = sanitizeHref(href);
+				if (!safeHref) return `<span>${escapeHTML(text ?? "")}</span>`;
+
+				const safeSrc = escapeHTML(safeHref);
+				const safeTitle = title ? ` title="${escapeHTML(title)}"` : "";
+				const safeAlt = escapeHTML(text ?? "");
+
+				if (isVideoUrl(safeHref)) {
+					return `<video controls${safeTitle}><source src="${safeSrc}">${safeAlt}</video>`;
+				}
+				if (isAudioUrl(safeHref)) {
+					return `<audio controls${safeTitle}><source src="${safeSrc}">${safeAlt}</audio>`;
+				}
+				return `<img src="${safeSrc}" alt="${safeAlt}"${safeTitle} />`;
+			},
+			html: (html) => sanitizeHtmlForMultimedia(html),
+		},
+		gfm: true,
+		breaks: true,
+	});
+}
+function isFencedBlockClosed(raw?: string): boolean {
+	if (!raw) return true;
+	/* eslint-disable-next-line no-control-regex */
+	const trimmed = raw.replace(/[\s\u0000]+$/, "");
+	const openingFenceMatch = trimmed.match(/^([`~]{3,})/);
+	if (!openingFenceMatch) {
+		return true;
+	}
+	const fence = openingFenceMatch[1];
+	const closingFencePattern = new RegExp(`(?:\n|\r\n)${fence}(?:[\t ]+)?$`);
+	return closingFencePattern.test(trimmed);
+}
+
+type CodeToken = {
+	type: "code";
+	lang: string;
+	code: string;
+	rawCode: string;
+	isClosed: boolean;
+};
+
+type TextToken = {
+	type: "text";
+	html: string | Promise<string>;
+};
+
+const blockCache = new Map<string, BlockToken>();
+
+function cacheKey(index: number, blockContent: string, sources: SimpleSource[]) {
+	const sourceKey = sources.map((s) => s.link).join("|");
+	return `${index}-${hashString(blockContent)}|${sourceKey}`;
+}
+
+export async function processTokens(content: string, sources: SimpleSource[]): Promise<Token[]> {
+	// Apply incomplete markdown preprocessing for smooth streaming
+	const processedContent = parseIncompleteMarkdown(content);
+
+	const marked = createMarkedInstance(sources);
+	const tokens = marked.lexer(processedContent);
+
+	const processedTokens = await Promise.all(
+		tokens.map(async (token) => {
+			if (token.type === "code") {
+				return {
+					type: "code" as const,
+					lang: token.lang,
+					code: highlightCode(token.text, token.lang),
+					rawCode: token.text,
+					isClosed: isFencedBlockClosed(token.raw ?? ""),
+				};
+			} else {
+				return {
+					type: "text" as const,
+					html: marked.parse(token.raw),
+				};
+			}
+		})
+	);
+
+	return processedTokens;
+}
+
+export function processTokensSync(content: string, sources: SimpleSource[]): Token[] {
+	// Apply incomplete markdown preprocessing for smooth streaming
+	const processedContent = parseIncompleteMarkdown(content);
+
+	const marked = createMarkedInstance(sources);
+	const tokens = marked.lexer(processedContent);
+	return tokens.map((token) => {
+		if (token.type === "code") {
+			return {
+				type: "code" as const,
+				lang: token.lang,
+				code: highlightCode(token.text, token.lang),
+				rawCode: token.text,
+				isClosed: isFencedBlockClosed(token.raw ?? ""),
+			};
+		}
+		return { type: "text" as const, html: marked.parse(token.raw) };
+	});
+}
+
+export type Token = CodeToken | TextToken;
+
+export type BlockToken = {
+	id: string;
+	content: string;
+	tokens: Token[];
+};
+
+/**
+ * Simple hash function for generating stable block IDs
+ */
+function hashString(str: string): string {
+	let hash = 0;
+	for (let i = 0; i < str.length; i++) {
+		const char = str.charCodeAt(i);
+		hash = (hash << 5) - hash + char;
+		hash = hash & hash; // Convert to 32bit integer
+	}
+	return Math.abs(hash).toString(36);
+}
+
+/**
+ * Process markdown content into blocks with stable IDs for efficient memoization.
+ * Each block is processed independently and assigned a content-based hash ID.
+ */
+export async function processBlocks(
+	content: string,
+	sources: SimpleSource[] = []
+): Promise<BlockToken[]> {
+	const blocks = parseMarkdownIntoBlocks(content);
+
+	return await Promise.all(
+		blocks.map(async (blockContent, index) => {
+			const key = cacheKey(index, blockContent, sources);
+			const cached = blockCache.get(key);
+			if (cached) return cached;
+
+			const tokens = await processTokens(blockContent, sources);
+			const block: BlockToken = {
+				id: `${index}-${hashString(blockContent)}`,
+				content: blockContent,
+				tokens,
+			};
+			blockCache.set(key, block);
+			return block;
+		})
+	);
+}
+
+/**
+ * Synchronous version of processBlocks for SSR
+ */
+export function processBlocksSync(content: string, sources: SimpleSource[] = []): BlockToken[] {
+	const blocks = parseMarkdownIntoBlocks(content);
+
+	return blocks.map((blockContent, index) => {
+		const key = cacheKey(index, blockContent, sources);
+		const cached = blockCache.get(key);
+		if (cached) return cached;
+
+		const tokens = processTokensSync(blockContent, sources);
+		const block: BlockToken = {
+			id: `${index}-${hashString(blockContent)}`,
+			content: blockContent,
+			tokens,
+		};
+		blockCache.set(key, block);
+		return block;
+	});
+}

package/src/ruvocal/src/lib/utils/mcpValidation.ts

@@ -0,0 +1,147 @@
+/**
+ * URL validation and sanitization utilities for MCP integration
+ */
+
+import { browser } from "$app/environment";
+import { dev } from "$app/environment";
+
+/**
+ * Sanitize and validate a URL for MCP server connections
+ * @param urlString - The URL string to validate
+ * @returns Sanitized URL string or null if invalid
+ */
+export function validateMcpServerUrl(urlString: string): string | null {
+	if (!urlString || typeof urlString !== "string") {
+		return null;
+	}
+
+	try {
+		const url = new URL(urlString.trim());
+
+		// Allow http/https only
+		if (!["http:", "https:"].includes(url.protocol)) {
+			return null;
+		}
+
+		// Warn about non-HTTPS in production
+		if (!dev && url.protocol === "http:" && browser) {
+			console.warn(
+				"Warning: Connecting to non-HTTPS MCP server in production. This may expose sensitive data."
+			);
+		}
+
+		// Block certain localhost/private IPs in production
+		if (!dev && isPrivateOrLocalhost(url.hostname)) {
+			console.warn("Warning: Localhost/private IP addresses are not recommended in production.");
+		}
+
+		return url.toString();
+	} catch (error) {
+		// Invalid URL
+		return null;
+	}
+}
+
+/**
+ * Check if hostname is localhost or a private IP
+ */
+function isPrivateOrLocalhost(hostname: string): boolean {
+	// Localhost checks
+	if (
+		hostname === "localhost" ||
+		hostname === "127.0.0.1" ||
+		hostname === "::1" ||
+		hostname.endsWith(".localhost")
+	) {
+		return true;
+	}
+
+	// Private IP ranges (IPv4)
+	const ipv4Regex = /^(10\.|172\.(1[6-9]|2[0-9]|3[01])\.|192\.168\.|127\.|0\.0\.0\.0|169\.254\.)/;
+	if (ipv4Regex.test(hostname)) {
+		return true;
+	}
+
+	return false;
+}
+
+/**
+ * Sanitize URL by removing sensitive parts
+ * Used for logging and display purposes
+ */
+export function sanitizeUrlForDisplay(urlString: string): string {
+	try {
+		const url = new URL(urlString);
+		// Remove username/password if present
+		url.username = "";
+		url.password = "";
+		return url.toString();
+	} catch {
+		return urlString;
+	}
+}
+
+/**
+ * Check if URL is safe to connect to
+ * Returns an error message if unsafe, null if safe
+ */
+export function checkUrlSafety(urlString: string): string | null {
+	const validated = validateMcpServerUrl(urlString);
+	if (!validated) {
+		return "Invalid URL. Please use http:// or https:// URLs only.";
+	}
+
+	try {
+		const url = new URL(validated);
+
+		// Additional safety checks
+		if (!dev && url.protocol === "http:") {
+			return "Non-HTTPS URLs are not recommended in production. Please use https:// for security.";
+		}
+
+		return null; // Safe
+	} catch {
+		return "Invalid URL format.";
+	}
+}
+
+/**
+ * Check if a header key is likely to contain sensitive data
+ */
+export function isSensitiveHeader(key: string): boolean {
+	const sensitiveKeys = [
+		"authorization",
+		"api-key",
+		"api_key",
+		"apikey",
+		"token",
+		"secret",
+		"password",
+		"bearer",
+		"x-api-key",
+		"x-auth-token",
+	];
+
+	const lowerKey = key.toLowerCase();
+	return sensitiveKeys.some((sensitive) => lowerKey.includes(sensitive));
+}
+
+/**
+ * Validate header key-value pair
+ * Returns error message if invalid, null if valid
+ */
+export function validateHeader(key: string, value: string): string | null {
+	if (!key || !key.trim()) {
+		return "Header name is required";
+	}
+
+	if (!/^[a-zA-Z0-9_-]+$/.test(key)) {
+		return "Header name can only contain letters, numbers, hyphens, and underscores";
+	}
+
+	if (!value) {
+		return "Header value is required";
+	}
+
+	return null;
+}

package/src/ruvocal/src/lib/utils/mergeAsyncGenerators.ts

@@ -0,0 +1,38 @@
+type Gen<T, TReturn> = AsyncGenerator<T, TReturn, undefined>;
+
+type GenPromiseMap<T, TReturn> = Map<
+	Gen<T, TReturn>,
+	Promise<{ gen: Gen<T, TReturn> } & IteratorResult<T, TReturn>>
+>;
+
+/** Merges multiple async generators into a single async generator that yields values from all of them in parallel. */
+export async function* mergeAsyncGenerators<T, TReturn>(
+	generators: Gen<T, TReturn>[]
+): Gen<T, TReturn[]> {
+	const promises: GenPromiseMap<T, TReturn> = new Map();
+	const results: Map<Gen<T, TReturn>, TReturn> = new Map();
+
+	for (const gen of generators) {
+		promises.set(
+			gen,
+			gen.next().then((result) => ({ gen, ...result }))
+		);
+	}
+
+	while (promises.size) {
+		const { gen, value, done } = await Promise.race(promises.values());
+		if (done) {
+			results.set(gen, value as TReturn);
+			promises.delete(gen);
+		} else {
+			promises.set(
+				gen,
+				gen.next().then((result) => ({ gen, ...result }))
+			);
+			yield value as T;
+		}
+	}
+
+	const orderedResults = generators.map((gen) => results.get(gen) as TReturn);
+	return orderedResults;
+}