visus-mcp 0.3.0 → 0.6.0

package/src/tools/search.ts

@@ -0,0 +1,263 @@
+/**
+ * Visus Search Tool - Safe Web Search
+ *
+ * Queries DuckDuckGo's Instant Answer API and sanitizes all results
+ * before returning them to the LLM.
+ *
+ * SECURITY: Every search result snippet and title passes through the
+ * sanitization pipeline. This prevents prompt injection via search results.
+ */
+
+import { sanitize } from '../sanitizer/index.js';
+import { generateThreatReport } from '../sanitizer/threat-reporter.js';
+import type { VisusSearchInput, VisusSearchOutput, Result } from '../types.js';
+import { Ok, Err } from '../types.js';
+
+/**
+ * DuckDuckGo API Response Types
+ */
+interface DuckDuckGoRelatedTopic {
+  Text?: string;
+  FirstURL?: string;
+}
+
+interface DuckDuckGoResponse {
+  AbstractText?: string;
+  AbstractURL?: string;
+  RelatedTopics?: Array<DuckDuckGoRelatedTopic | { Topics: DuckDuckGoRelatedTopic[] }>;
+}
+
+/**
+ * Search the web via DuckDuckGo and return sanitized results
+ *
+ * @param input Search query and options
+ * @returns Sanitized search results with injection detection metadata
+ */
+export async function visusSearch(input: VisusSearchInput): Promise<Result<VisusSearchOutput, Error>> {
+  // Validate input
+  if (!input.query || typeof input.query !== 'string' || input.query.trim().length === 0) {
+    return Err(new Error('query must be a non-empty string'));
+  }
+
+  // Enforce max_results cap
+  const maxResults = Math.min(input.max_results ?? 5, 10);
+
+  try {
+    // Call DuckDuckGo Instant Answer API
+    const query = encodeURIComponent(input.query.trim());
+    const apiUrl = `https://api.duckduckgo.com/?q=${query}&format=json&no_redirect=1&no_html=1`;
+
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), 8000);
+
+    let response: Response;
+    try {
+      response = await fetch(apiUrl, {
+        signal: controller.signal,
+        headers: {
+          'User-Agent': 'visus-mcp/0.3.0 (https://github.com/lateos/visus-mcp)'
+        }
+      });
+    } finally {
+      clearTimeout(timeout);
+    }
+
+    if (!response.ok) {
+      return Ok({
+        query: input.query,
+        result_count: 0,
+        sanitized: true,
+        results: [],
+        total_injections_removed: 0,
+        message: `Search unavailable (HTTP ${response.status})`
+      });
+    }
+
+    const data = await response.json() as DuckDuckGoResponse;
+
+    // Extract results from DuckDuckGo response
+    const rawResults: Array<{ title: string; url: string; snippet: string }> = [];
+
+    // Add AbstractText as first result if present
+    if (data.AbstractText && data.AbstractURL) {
+      rawResults.push({
+        title: extractTitle(data.AbstractText),
+        url: data.AbstractURL,
+        snippet: data.AbstractText
+      });
+    }
+
+    // Extract from RelatedTopics
+    if (data.RelatedTopics) {
+      for (const topic of data.RelatedTopics) {
+        // Handle both direct topics and nested topic groups
+        if ('Topics' in topic && Array.isArray(topic.Topics)) {
+          // Nested topics group
+          for (const nestedTopic of topic.Topics) {
+            if (nestedTopic.Text && nestedTopic.FirstURL) {
+              rawResults.push({
+                title: extractTitle(nestedTopic.Text),
+                url: nestedTopic.FirstURL,
+                snippet: nestedTopic.Text
+              });
+            }
+          }
+        } else if ('Text' in topic && topic.Text && topic.FirstURL) {
+          // Direct topic
+          rawResults.push({
+            title: extractTitle(topic.Text),
+            url: topic.FirstURL,
+            snippet: topic.Text
+          });
+        }
+
+        // Stop if we've collected enough results
+        if (rawResults.length >= maxResults) {
+          break;
+        }
+      }
+    }
+
+    // Filter out results with empty URLs and limit to max_results
+    const validResults = rawResults
+      .filter(r => r.url && r.url.trim().length > 0)
+      .slice(0, maxResults);
+
+    // If no results found, return empty array with message
+    if (validResults.length === 0) {
+      return Ok({
+        query: input.query,
+        result_count: 0,
+        sanitized: true,
+        results: [],
+        total_injections_removed: 0,
+        message: 'No results found'
+      });
+    }
+
+    // Sanitize each result independently
+    const sanitizedResults = [];
+    const allPatternsDetected = new Set<string>();
+    let totalInjectionsRemoved = 0;
+    let totalPIIRedacted = 0;
+
+    for (const result of validResults) {
+      // Sanitize title
+      const titleSanitization = sanitize(result.title);
+
+      // Sanitize snippet
+      const snippetSanitization = sanitize(result.snippet);
+
+      const injectionsRemoved =
+        titleSanitization.sanitization.patterns_detected.length +
+        snippetSanitization.sanitization.patterns_detected.length;
+
+      const piiRedacted =
+        titleSanitization.sanitization.pii_types_redacted.length +
+        snippetSanitization.sanitization.pii_types_redacted.length;
+
+      totalInjectionsRemoved += injectionsRemoved;
+      totalPIIRedacted += piiRedacted;
+
+      // Collect all patterns detected across all results
+      titleSanitization.sanitization.patterns_detected.forEach(p => allPatternsDetected.add(p));
+      snippetSanitization.sanitization.patterns_detected.forEach(p => allPatternsDetected.add(p));
+
+      sanitizedResults.push({
+        title: titleSanitization.content,
+        url: result.url,
+        snippet: snippetSanitization.content,
+        injections_removed: injectionsRemoved,
+        pii_redacted: piiRedacted
+      });
+    }
+
+    // Generate aggregated threat report for all search results
+    const threatReport = generateThreatReport({
+      patterns_detected: Array.from(allPatternsDetected),
+      pii_redacted: totalPIIRedacted,
+      source_url: `DuckDuckGo Search: ${input.query}`
+    });
+
+    return Ok({
+      query: input.query,
+      result_count: sanitizedResults.length,
+      sanitized: true,
+      results: sanitizedResults,
+      total_injections_removed: totalInjectionsRemoved,
+      // Include threat_report only if findings exist
+      ...(threatReport && { threat_report: threatReport })
+    });
+
+  } catch (error) {
+    // Handle timeout or network errors
+    if (error instanceof Error && error.name === 'AbortError') {
+      return Ok({
+        query: input.query,
+        result_count: 0,
+        sanitized: true,
+        results: [],
+        total_injections_removed: 0,
+        message: 'Search unavailable (timeout)'
+      });
+    }
+
+    return Ok({
+      query: input.query,
+      result_count: 0,
+      sanitized: true,
+      results: [],
+      total_injections_removed: 0,
+      message: `Search unavailable: ${error instanceof Error ? error.message : String(error)}`
+    });
+  }
+}
+
+/**
+ * Extract title from text (first sentence or up to 80 chars)
+ */
+function extractTitle(text: string): string {
+  // Try to find first sentence
+  const firstSentenceMatch = text.match(/^[^.!?]+[.!?]/);
+  if (firstSentenceMatch) {
+    const sentence = firstSentenceMatch[0].trim();
+    if (sentence.length <= 80) {
+      return sentence;
+    }
+  }
+
+  // Fallback to first 80 chars
+  if (text.length <= 80) {
+    return text.trim();
+  }
+
+  return text.substring(0, 77).trim() + '...';
+}
+
+/**
+ * Tool definition for MCP registration
+ */
+export const visusSearchToolDefinition = {
+  name: 'visus_search',
+  title: 'Search the Web (Sanitized)',
+  description: 'Searches the web via DuckDuckGo and returns sanitized results with prompt injection and PII removed before reaching the LLM. Use before visus_fetch or visus_read to safely discover and then read pages.',
+  inputSchema: {
+    type: 'object',
+    properties: {
+      query: {
+        type: 'string',
+        description: 'Search query'
+      },
+      max_results: {
+        type: 'number',
+        description: 'Maximum number of results to return (default: 5, max: 10)',
+        default: 5
+      }
+    },
+    required: ['query']
+  },
+  readOnlyHint: true,
+  destructiveHint: false,
+  idempotentHint: true,
+  openWorldHint: true
+};

package/src/types.ts

@@ -2,6 +2,8 @@
  * Shared TypeScript interfaces for Visus MCP tool
  */
 
+import type { ThreatReport } from './sanitizer/threat-reporter.js';
+
 /**
  * Input options for visus_fetch tool
  */
@@ -28,7 +30,12 @@ export interface VisusFetchOutput {
     fetched_at: string;
     content_length_original: number;
     content_length_sanitized: number;
+    format_detected?: 'html' | 'json' | 'xml' | 'rss';
+    content_type?: string;
+    truncated?: boolean;
+    truncated_at_chars?: number;
   };
+  threat_report?: ThreatReport;
 }
 
 /**
@@ -40,6 +47,14 @@ export interface VisusFetchStructuredInput {
   timeout_ms?: number;
 }
 
+/**
+ * Input for visus_read tool
+ */
+export interface VisusReadInput {
+  url: string;
+  timeout_ms?: number;
+}
+
 /**
  * Output from visus_fetch_structured tool
  */
@@ -57,7 +72,60 @@ export interface VisusFetchStructuredOutput {
     fetched_at: string;
     content_length_original: number;
     content_length_sanitized: number;
+    format_detected?: 'html' | 'json' | 'xml' | 'rss';
+    content_type?: string;
+    truncated?: boolean;
+    truncated_at_chars?: number;
   };
+  threat_report?: ThreatReport;
+}
+
+/**
+ * Output from visus_read tool
+ */
+export interface VisusReadOutput {
+  url: string;
+  content: string;
+  metadata: {
+    title: string;
+    author: string | null;
+    published: string | null;
+    word_count: number;
+    reader_mode_available: boolean;
+    sanitized: true;
+    injections_removed: number;
+    pii_redacted: number;
+    truncated: boolean;
+    fetched_at?: string;
+  };
+  threat_report?: ThreatReport;
+}
+
+/**
+ * Input for visus_search tool
+ */
+export interface VisusSearchInput {
+  query: string;
+  max_results?: number;
+}
+
+/**
+ * Output from visus_search tool
+ */
+export interface VisusSearchOutput {
+  query: string;
+  result_count: number;
+  sanitized: true;
+  results: Array<{
+    title: string;
+    url: string;
+    snippet: string;
+    injections_removed: number;
+    pii_redacted: number;
+  }>;
+  total_injections_removed: number;
+  message?: string;
+  threat_report?: ThreatReport;
 }
 
 /**
@@ -67,6 +135,7 @@ export interface BrowserRenderResult {
   html: string;
   title: string;
   url: string;
+  contentType?: string;
   text?: string;
   error?: string;
 }

package/src/utils/format-converter.ts

@@ -0,0 +1,236 @@
+/**
+ * Format Converter - Content-Type based format detection and conversion
+ *
+ * Handles format-appropriate conversion based on detected Content-Type.
+ * Supports HTML, JSON, XML, and RSS/Atom feeds.
+ */
+
+import { XMLParser } from 'fast-xml-parser';
+
+/**
+ * Detected format type
+ */
+export type FormatType = 'html' | 'json' | 'xml' | 'rss';
+
+/**
+ * Detect format from Content-Type header
+ *
+ * @param contentType - Content-Type header value (e.g., "application/json", "text/html; charset=utf-8")
+ * @returns Detected format type
+ */
+export function detectFormat(contentType: string): FormatType {
+  // Normalize: lowercase and extract MIME type (before semicolon)
+  const mimeType = contentType.toLowerCase().split(';')[0].trim();
+
+  // HTML formats
+  if (mimeType === 'text/html' || mimeType === 'application/xhtml+xml') {
+    return 'html';
+  }
+
+  // JSON formats
+  if (mimeType === 'application/json' || mimeType === 'text/json') {
+    return 'json';
+  }
+
+  // RSS/Atom feed formats
+  if (mimeType === 'application/rss+xml' ||
+      mimeType === 'application/atom+xml' ||
+      mimeType === 'application/feed+json') {
+    return 'rss';
+  }
+
+  // XML formats (must come after RSS check)
+  if (mimeType === 'application/xml' || mimeType === 'text/xml') {
+    return 'xml';
+  }
+
+  // Default to HTML for unknown types
+  return 'html';
+}
+
+/**
+ * Convert JSON content to formatted string
+ *
+ * @param raw - Raw JSON string
+ * @returns Formatted JSON string with prefix, or raw string if parse fails
+ */
+export function convertJson(raw: string): string {
+  try {
+    // Parse and re-stringify with 2-space indent for readability
+    const parsed = JSON.parse(raw);
+    const formatted = JSON.stringify(parsed, null, 2);
+    return `JSON Response:\n\n${formatted}`;
+  } catch (error) {
+    // Parse failed, return raw string unchanged
+    return raw;
+  }
+}
+
+/**
+ * Convert XML content to clean text representation
+ *
+ * @param raw - Raw XML string
+ * @returns Formatted XML representation with prefix, or tag-stripped fallback if parse fails
+ */
+export function convertXml(raw: string): string {
+  try {
+    const parser = new XMLParser({
+      ignoreAttributes: false,
+      attributeNamePrefix: '@_',
+      textNodeName: '#text',
+      ignoreDeclaration: true,
+      ignorePiTags: true,
+      removeNSPrefix: true,
+    });
+
+    const parsed = parser.parse(raw);
+    const formatted = JSON.stringify(parsed, null, 2);
+
+    return `XML Response:\n\n${formatted}`;
+  } catch (error) {
+    // Parse failed, strip XML tags using regex and return
+    const stripped = raw.replace(/<[^>]+>/g, '').trim();
+    return `XML Response:\n\n${stripped}`;
+  }
+}
+
+/**
+ * Convert RSS/Atom feed content to clean Markdown
+ *
+ * @param raw - Raw RSS/Atom XML string
+ * @returns Formatted Markdown representation, or falls back to convertXml if parse fails
+ */
+export function convertRss(raw: string): string {
+  try {
+    const parser = new XMLParser({
+      ignoreAttributes: false,
+      attributeNamePrefix: '@_',
+      textNodeName: '#text',
+      removeNSPrefix: true,
+    });
+
+    const parsed = parser.parse(raw);
+
+    // Handle RSS 2.0 format
+    if (parsed.rss && parsed.rss.channel) {
+      return formatRss2(parsed.rss.channel);
+    }
+
+    // Handle Atom format
+    if (parsed.feed) {
+      return formatAtom(parsed.feed);
+    }
+
+    // Handle RSS 1.0 (RDF) format
+    if (parsed.rdf && parsed.rdf.channel) {
+      return formatRss2(parsed.rdf.channel);
+    }
+
+    // Unknown feed format, fall back to XML
+    return convertXml(raw);
+
+  } catch (error) {
+    // Parse failed, fall back to XML converter
+    return convertXml(raw);
+  }
+}
+
+/**
+ * Format RSS 2.0 feed data as Markdown
+ */
+function formatRss2(channel: any): string {
+  const title = channel.title || 'Untitled Feed';
+  const description = channel.description || '';
+  const items = Array.isArray(channel.item) ? channel.item : (channel.item ? [channel.item] : []);
+
+  let markdown = `RSS Feed:\n\n# ${title}\n`;
+
+  if (description) {
+    markdown += `${description}\n`;
+  }
+
+  markdown += '\n## Items\n\n';
+
+  // Extract up to 10 items
+  const itemsToShow = items.slice(0, 10);
+
+  for (const item of itemsToShow) {
+    const itemTitle = item.title || 'Untitled';
+    const itemLink = item.link || '';
+    const itemDescription = item.description || '';
+    const itemPubDate = item.pubDate || '';
+
+    // Truncate description to 200 chars
+    const truncatedDesc = itemDescription.length > 200
+      ? itemDescription.substring(0, 200) + '...'
+      : itemDescription;
+
+    markdown += `### ${itemTitle}\n\n`;
+
+    if (truncatedDesc) {
+      markdown += `${truncatedDesc}\n\n`;
+    }
+
+    if (itemLink) {
+      markdown += `Link: ${itemLink}\n`;
+    }
+
+    if (itemPubDate) {
+      markdown += `Published: ${itemPubDate}\n`;
+    }
+
+    markdown += '\n---\n\n';
+  }
+
+  return markdown;
+}
+
+/**
+ * Format Atom feed data as Markdown
+ */
+function formatAtom(feed: any): string {
+  const title = feed.title || 'Untitled Feed';
+  const subtitle = feed.subtitle || '';
+  const entries = Array.isArray(feed.entry) ? feed.entry : (feed.entry ? [feed.entry] : []);
+
+  let markdown = `RSS Feed:\n\n# ${title}\n`;
+
+  if (subtitle) {
+    markdown += `${subtitle}\n`;
+  }
+
+  markdown += '\n## Items\n\n';
+
+  // Extract up to 10 entries
+  const entriesToShow = entries.slice(0, 10);
+
+  for (const entry of entriesToShow) {
+    const entryTitle = entry.title || 'Untitled';
+    const entryLink = entry.link ? (entry.link['@_href'] || entry.link) : '';
+    const entrySummary = entry.summary || entry.content || '';
+    const entryPublished = entry.published || entry.updated || '';
+
+    // Truncate summary to 200 chars
+    const truncatedSummary = entrySummary.length > 200
+      ? entrySummary.substring(0, 200) + '...'
+      : entrySummary;
+
+    markdown += `### ${entryTitle}\n\n`;
+
+    if (truncatedSummary) {
+      markdown += `${truncatedSummary}\n\n`;
+    }
+
+    if (entryLink) {
+      markdown += `Link: ${entryLink}\n`;
+    }
+
+    if (entryPublished) {
+      markdown += `Published: ${entryPublished}\n`;
+    }
+
+    markdown += '\n---\n\n';
+  }
+
+  return markdown;
+}

package/src/utils/truncate.ts

@@ -0,0 +1,64 @@
+/**
+ * Token-aware content truncation utility
+ *
+ * Anthropic MCP Directory enforces a 25,000 token response limit.
+ * This utility provides safe truncation with token estimation.
+ */
+
+/**
+ * Maximum tokens allowed in MCP response (Anthropic Directory limit)
+ * We target 24,000 to leave headroom for metadata/JSON structure
+ */
+const MAX_TOKENS = 24000;
+
+/**
+ * Conservative token estimation: 1 token ≈ 4 characters
+ * This is a safe approximation that errs on the side of caution
+ */
+const CHARS_PER_TOKEN = 4;
+
+/**
+ * Maximum characters based on token limit
+ */
+const MAX_CHARS = MAX_TOKENS * CHARS_PER_TOKEN; // 96,000 characters
+
+/**
+ * Truncate content if it exceeds the token ceiling
+ *
+ * @param content Content to potentially truncate
+ * @returns Truncated content and metadata
+ */
+export function truncateContent(content: string): {
+  content: string;
+  truncated: boolean;
+  truncated_at_chars?: number;
+} {
+  if (content.length <= MAX_CHARS) {
+    // Content is within limits
+    return {
+      content,
+      truncated: false
+    };
+  }
+
+  // Content exceeds limit - truncate with warning message
+  const truncatedContent = content.substring(0, MAX_CHARS);
+  const warningMessage = `\n\n--- CONTENT TRUNCATED ---\nOriginal length: ${content.length} characters (~${Math.ceil(content.length / CHARS_PER_TOKEN)} tokens)\nTruncated to: ${MAX_CHARS} characters (~${MAX_TOKENS} tokens)\nReason: Anthropic MCP Directory enforces a 25,000 token response limit\n`;
+
+  return {
+    content: truncatedContent + warningMessage,
+    truncated: true,
+    truncated_at_chars: MAX_CHARS
+  };
+}
+
+/**
+ * Estimate token count for a given string
+ * Uses conservative 4 chars per token approximation
+ *
+ * @param text Text to estimate
+ * @returns Estimated token count
+ */
+export function estimateTokens(text: string): number {
+  return Math.ceil(text.length / CHARS_PER_TOKEN);
+}