pse-mcp 0.1.0

package/dist-package/dist/google-search.js

@@ -0,0 +1,420 @@
+import { Server } from '@modelcontextprotocol/sdk/server/index.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
+import { GoogleSearchService } from './services/google-search.service.js';
+import { ContentExtractor } from './services/content-extractor.service.js';
+class GoogleSearchServer {
+    constructor() {
+        this.searchService = new GoogleSearchService();
+        this.contentExtractor = new ContentExtractor();
+        this.server = new Server({
+            name: 'google-search',
+            version: '1.0.0'
+        }, {
+            capabilities: {
+                tools: {
+                    google_search: {
+                        description: 'Search Google and return relevant results from the web. This tool finds web pages, articles, and information on specific topics using Google\'s search engine. Results include titles, snippets, and URLs that can be analyzed further using extract_webpage_content.',
+                        inputSchema: {
+                            type: 'object',
+                            properties: {
+                                query: {
+                                    type: 'string',
+                                    description: 'Search query - be specific and use quotes for exact matches. For best results, use clear keywords and avoid very long queries.'
+                                },
+                                num_results: {
+                                    type: 'number',
+                                    description: 'Number of results to return (default: 5, max: 10). Increase for broader coverage, decrease for faster response.'
+                                },
+                                site: {
+                                    type: 'string',
+                                    description: 'Limit search results to a specific website domain (e.g., "wikipedia.org" or "nytimes.com").'
+                                },
+                                language: {
+                                    type: 'string',
+                                    description: 'Filter results by language using ISO 639-1 codes (e.g., "en" for English, "es" for Spanish, "fr" for French).'
+                                },
+                                dateRestrict: {
+                                    type: 'string',
+                                    description: 'Filter results by date using Google\'s date restriction format: "d[number]" for past days, "w[number]" for past weeks, "m[number]" for past months, or "y[number]" for past years. Example: "m6" for results from the past 6 months.'
+                                },
+                                exactTerms: {
+                                    type: 'string',
+                                    description: 'Search for results that contain this exact phrase. This is equivalent to putting the terms in quotes in the search query.'
+                                },
+                                resultType: {
+                                    type: 'string',
+                                    description: 'Specify the type of results to return. Options include "image" (or "images"), "news", and "video" (or "videos"). Default is general web results.'
+                                },
+                                page: {
+                                    type: 'number',
+                                    description: 'Page number for paginated results (starts at 1). Use in combination with resultsPerPage to navigate through large result sets.'
+                                },
+                                resultsPerPage: {
+                                    type: 'number',
+                                    description: 'Number of results to show per page (default: 5, max: 10). Controls how many results are returned for each page.'
+                                },
+                                sort: {
+                                    type: 'string',
+                                    description: 'Sorting method for search results. Options: "relevance" (default) or "date" (most recent first).'
+                                }
+                            },
+                            required: ['query']
+                        }
+                    },
+                    extract_webpage_content: {
+                        description: 'Extract and analyze content from a webpage, converting it to readable text. This tool fetches the main content while removing ads, navigation elements, and other clutter. Use it to get detailed information from specific pages found via google_search. Works with most common webpage formats including articles, blogs, and documentation.',
+                        inputSchema: {
+                            type: 'object',
+                            properties: {
+                                url: {
+                                    type: 'string',
+                                    description: 'Full URL of the webpage to extract content from (must start with http:// or https://). Ensure the URL is from a public webpage and not behind authentication.'
+                                },
+                                format: {
+                                    type: 'string',
+                                    description: 'Output format for the extracted content. Options: "markdown" (default), "html", or "text".'
+                                }
+                            },
+                            required: ['url']
+                        }
+                    },
+                    extract_multiple_webpages: {
+                        description: 'Extract and analyze content from multiple webpages in a single request. This tool is ideal for comparing information across different sources or gathering comprehensive information on a topic. Limited to 5 URLs per request to maintain performance.',
+                        inputSchema: {
+                            type: 'object',
+                            properties: {
+                                urls: {
+                                    type: 'array',
+                                    items: { type: 'string' },
+                                    description: 'Array of webpage URLs to extract content from. Each URL must be public and start with http:// or https://. Maximum 5 URLs per request.'
+                                },
+                                format: {
+                                    type: 'string',
+                                    description: 'Output format for the extracted content. Options: "markdown" (default), "html", or "text".'
+                                }
+                            },
+                            required: ['urls']
+                        }
+                    }
+                }
+            }
+        });
+        // Register tool list handler
+        this.server.setRequestHandler(ListToolsRequestSchema, async () => ({
+            tools: [
+                {
+                    name: 'google_search',
+                    description: 'Search Google and return relevant results from the web. This tool finds web pages, articles, and information on specific topics using Google\'s search engine. Results include titles, snippets, and URLs that can be analyzed further using extract_webpage_content.',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            query: {
+                                type: 'string',
+                                description: 'Search query - be specific and use quotes for exact matches. For best results, use clear keywords and avoid very long queries.'
+                            },
+                            num_results: {
+                                type: 'number',
+                                description: 'Number of results to return (default: 5, max: 10). Increase for broader coverage, decrease for faster response.'
+                            },
+                            site: {
+                                type: 'string',
+                                description: 'Limit search results to a specific website domain (e.g., "wikipedia.org" or "nytimes.com").'
+                            },
+                            language: {
+                                type: 'string',
+                                description: 'Filter results by language using ISO 639-1 codes (e.g., "en" for English, "es" for Spanish, "fr" for French).'
+                            },
+                            dateRestrict: {
+                                type: 'string',
+                                description: 'Filter results by date using Google\'s date restriction format: "d[number]" for past days, "w[number]" for past weeks, "m[number]" for past months, or "y[number]" for past years. Example: "m6" for results from the past 6 months.'
+                            },
+                            exactTerms: {
+                                type: 'string',
+                                description: 'Search for results that contain this exact phrase. This is equivalent to putting the terms in quotes in the search query.'
+                            },
+                            resultType: {
+                                type: 'string',
+                                description: 'Specify the type of results to return. Options include "image" (or "images"), "news", and "video" (or "videos"). Default is general web results.'
+                            },
+                            page: {
+                                type: 'number',
+                                description: 'Page number for paginated results (starts at 1). Use in combination with resultsPerPage to navigate through large result sets.'
+                            },
+                            resultsPerPage: {
+                                type: 'number',
+                                description: 'Number of results to show per page (default: 5, max: 10). Controls how many results are returned for each page.'
+                            },
+                            sort: {
+                                type: 'string',
+                                description: 'Sorting method for search results. Options: "relevance" (default) or "date" (most recent first).'
+                            }
+                        },
+                        required: ['query']
+                    }
+                },
+                {
+                    name: 'extract_webpage_content',
+                    description: 'Extract and analyze content from a webpage, converting it to readable text. This tool fetches the main content while removing ads, navigation elements, and other clutter. Use it to get detailed information from specific pages found via google_search. Works with most common webpage formats including articles, blogs, and documentation.',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            url: {
+                                type: 'string',
+                                description: 'Full URL of the webpage to extract content from (must start with http:// or https://). Ensure the URL is from a public webpage and not behind authentication.'
+                            },
+                            format: {
+                                type: 'string',
+                                description: 'Output format for the extracted content. Options: "markdown" (default), "html", or "text".'
+                            }
+                        },
+                        required: ['url']
+                    }
+                },
+                {
+                    name: 'extract_multiple_webpages',
+                    description: 'Extract and analyze content from multiple webpages in a single request. This tool is ideal for comparing information across different sources or gathering comprehensive information on a topic. Limited to 5 URLs per request to maintain performance.',
+                    inputSchema: {
+                        type: 'object',
+                        properties: {
+                            urls: {
+                                type: 'array',
+                                items: { type: 'string' },
+                                description: 'Array of webpage URLs to extract content from. Each URL must be public and start with http:// or https://. Maximum 5 URLs per request.'
+                            },
+                            format: {
+                                type: 'string',
+                                description: 'Output format for the extracted content. Options: "markdown" (default), "html", or "text".'
+                            }
+                        },
+                        required: ['urls']
+                    }
+                }
+            ]
+        }));
+        // Register tool call handler
+        this.server.setRequestHandler(CallToolRequestSchema, async (request) => {
+            switch (request.params.name) {
+                case 'google_search':
+                    if (typeof request.params.arguments === 'object' && request.params.arguments !== null && 'query' in request.params.arguments) {
+                        return this.handleSearch({
+                            query: String(request.params.arguments.query),
+                            num_results: typeof request.params.arguments.num_results === 'number' ? request.params.arguments.num_results : undefined,
+                            filters: {
+                                site: request.params.arguments.site ? String(request.params.arguments.site) : undefined,
+                                language: request.params.arguments.language ? String(request.params.arguments.language) : undefined,
+                                dateRestrict: request.params.arguments.dateRestrict ? String(request.params.arguments.dateRestrict) : undefined,
+                                exactTerms: request.params.arguments.exactTerms ? String(request.params.arguments.exactTerms) : undefined,
+                                resultType: request.params.arguments.resultType ? String(request.params.arguments.resultType) : undefined,
+                                page: typeof request.params.arguments.page === 'number' ? request.params.arguments.page : undefined,
+                                resultsPerPage: typeof request.params.arguments.resultsPerPage === 'number' ? request.params.arguments.resultsPerPage : undefined,
+                                sort: request.params.arguments.sort ? String(request.params.arguments.sort) : undefined
+                            }
+                        });
+                    }
+                    throw new Error('Invalid arguments for google_search tool');
+                case 'extract_webpage_content':
+                    if (typeof request.params.arguments === 'object' && request.params.arguments !== null && 'url' in request.params.arguments) {
+                        return this.handleAnalyzeWebpage({
+                            url: String(request.params.arguments.url),
+                            format: request.params.arguments.format ? String(request.params.arguments.format) : 'markdown'
+                        });
+                    }
+                    throw new Error('Invalid arguments for extract_webpage_content tool');
+                case 'extract_multiple_webpages':
+                    if (typeof request.params.arguments === 'object' && request.params.arguments !== null && 'urls' in request.params.arguments && Array.isArray(request.params.arguments.urls)) {
+                        return this.handleBatchAnalyzeWebpages({
+                            urls: request.params.arguments.urls.map(String),
+                            format: request.params.arguments.format ? String(request.params.arguments.format) : 'markdown'
+                        });
+                    }
+                    throw new Error('Invalid arguments for extract_multiple_webpages tool');
+                default:
+                    throw new Error(`Unknown tool: ${request.params.name}`);
+            }
+        });
+    }
+    async handleSearch(args) {
+        try {
+            const { results, pagination, categories } = await this.searchService.search(args.query, args.num_results, args.filters);
+            if (results.length === 0) {
+                return {
+                    content: [{
+                            type: 'text',
+                            text: 'No results found. Try:\n- Using different keywords\n- Removing quotes from non-exact phrases\n- Using more general terms'
+                        }],
+                    isError: true
+                };
+            }
+            // Format results in a more concise, readable way
+            const formattedResults = results.map(result => ({
+                title: result.title,
+                link: result.link,
+                snippet: result.snippet,
+                category: result.category
+            }));
+            // Format results in a more AI-friendly way
+            let responseText = `Search results for "${args.query}":\n\n`;
+            // Add category summary if available
+            if (categories && categories.length > 0) {
+                responseText += "Categories: " + categories.map(c => `${c.name} (${c.count})`).join(', ') + "\n\n";
+            }
+            // Add pagination info
+            if (pagination) {
+                responseText += `Showing page ${pagination.currentPage}${pagination.totalResults ? ` of approximately ${pagination.totalResults} results` : ''}\n\n`;
+            }
+            // Add each result in a readable format
+            formattedResults.forEach((result, index) => {
+                responseText += `${index + 1}. ${result.title}\n`;
+                responseText += `   URL: ${result.link}\n`;
+                responseText += `   ${result.snippet}\n\n`;
+            });
+            // Add navigation hints if pagination exists
+            if (pagination && (pagination.hasNextPage || pagination.hasPreviousPage)) {
+                responseText += "Navigation: ";
+                if (pagination.hasPreviousPage) {
+                    responseText += "Use 'page: " + (pagination.currentPage - 1) + "' for previous results. ";
+                }
+                if (pagination.hasNextPage) {
+                    responseText += "Use 'page: " + (pagination.currentPage + 1) + "' for more results.";
+                }
+                responseText += "\n";
+            }
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: responseText,
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : 'Unknown error during search';
+            return {
+                content: [{ type: 'text', text: message }],
+                isError: true
+            };
+        }
+    }
+    async handleAnalyzeWebpage(args) {
+        try {
+            const content = await this.contentExtractor.extractContent(args.url, args.format);
+            // Format the response in a more readable, concise way
+            let responseText = `Content from: ${content.url}\n\n`;
+            responseText += `Title: ${content.title}\n`;
+            if (content.description) {
+                responseText += `Description: ${content.description}\n`;
+            }
+            responseText += `\nStats: ${content.stats.word_count} words, ${content.stats.approximate_chars} characters\n\n`;
+            // Add the summary if available
+            if (content.summary) {
+                responseText += `Summary: ${content.summary}\n\n`;
+            }
+            // Add a preview of the content
+            responseText += `Content Preview:\n${content.content_preview.first_500_chars}\n\n`;
+            // Add a note about requesting specific information
+            responseText += `Note: This is a preview of the content. For specific information, please ask about particular aspects of this webpage.`;
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: responseText,
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred';
+            const helpText = 'Common issues:\n- Check if the URL is accessible in a browser\n- Ensure the webpage is public\n- Try again if it\'s a temporary network issue';
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: `${errorMessage}\n\n${helpText}`,
+                    },
+                ],
+                isError: true,
+            };
+        }
+    }
+    async handleBatchAnalyzeWebpages(args) {
+        if (args.urls.length > 5) {
+            return {
+                content: [{
+                        type: 'text',
+                        text: 'Maximum 5 URLs allowed per request to maintain performance. Please reduce the number of URLs.'
+                    }],
+                isError: true
+            };
+        }
+        try {
+            const results = await this.contentExtractor.batchExtractContent(args.urls, args.format);
+            // Format the response in a more readable, concise way
+            let responseText = `Content from ${args.urls.length} webpages:\n\n`;
+            for (const [url, result] of Object.entries(results)) {
+                responseText += `URL: ${url}\n`;
+                if ('error' in result) {
+                    responseText += `Error: ${result.error}\n\n`;
+                    continue;
+                }
+                responseText += `Title: ${result.title}\n`;
+                if (result.description) {
+                    responseText += `Description: ${result.description}\n`;
+                }
+                responseText += `Stats: ${result.stats.word_count} words\n`;
+                // Add summary if available
+                if (result.summary) {
+                    responseText += `Summary: ${result.summary}\n`;
+                }
+                responseText += `Preview: ${result.content_preview.first_500_chars.substring(0, 150)}...\n\n`;
+            }
+            responseText += `Note: These are previews of the content. To analyze the full content of a specific URL, use the extract_webpage_content tool with that URL.`;
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: responseText,
+                    },
+                ],
+            };
+        }
+        catch (error) {
+            const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred';
+            const helpText = 'Common issues:\n- Check if all URLs are accessible in a browser\n- Ensure all webpages are public\n- Try again if it\'s a temporary network issue\n- Consider reducing the number of URLs';
+            return {
+                content: [
+                    {
+                        type: 'text',
+                        text: `${errorMessage}\n\n${helpText}`,
+                    },
+                ],
+                isError: true,
+            };
+        }
+    }
+    async start() {
+        try {
+            const transport = new StdioServerTransport();
+            await this.server.connect(transport);
+            console.error('Google Search MCP server running');
+            // Keep the process running
+            process.on('SIGINT', () => {
+                this.server.close().catch(console.error);
+                process.exit(0);
+            });
+        }
+        catch (error) {
+            if (error instanceof Error) {
+                console.error('Failed to start MCP server:', error.message);
+            }
+            else {
+                console.error('Failed to start MCP server: Unknown error');
+            }
+            process.exit(1);
+        }
+    }
+}
+// Start the server
+const server = new GoogleSearchServer();
+server.start().catch(console.error);

package/dist-package/dist/services/content-extractor.service.js

@@ -0,0 +1,195 @@
+import axios from 'axios';
+import * as cheerio from 'cheerio';
+import { Readability } from '@mozilla/readability';
+import { JSDOM } from 'jsdom';
+import MarkdownIt from 'markdown-it';
+import TurndownService from 'turndown';
+export class ContentExtractor {
+    constructor() {
+        // Cache for webpage content (key: url + format, value: content)
+        this.contentCache = new Map();
+        // Cache expiration time in milliseconds (30 minutes)
+        this.cacheTTL = 30 * 60 * 1000;
+        this.md = new MarkdownIt();
+        this.turndownService = new TurndownService({
+            headingStyle: 'atx',
+            codeBlockStyle: 'fenced'
+        });
+    }
+    cleanText(text) {
+        // Remove multiple blank lines
+        text = text.replace(/\n\s*\n\s*\n/g, '\n\n');
+        // Remove excessive spaces
+        text = text.replace(/ +/g, ' ');
+        return text.trim();
+    }
+    cleanMarkdown(text) {
+        let cleanedText = this.cleanText(text);
+        // Ensure headers have space after #
+        cleanedText = cleanedText.replace(/#([A-Za-z0-9])/g, '# $1');
+        return cleanedText;
+    }
+    htmlToMarkdown(html) {
+        return this.cleanMarkdown(this.turndownService.turndown(html));
+    }
+    htmlToPlainText(html) {
+        const dom = new JSDOM(html);
+        return this.cleanText(dom.window.document.body.textContent || '');
+    }
+    isValidUrl(url) {
+        try {
+            new URL(url);
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Generate a cache key from URL and format
+     */
+    generateCacheKey(url, format) {
+        return `${url}|${format}`;
+    }
+    /**
+     * Check if a cache entry is still valid
+     */
+    isCacheValid(entry) {
+        const now = Date.now();
+        return now - entry.timestamp < this.cacheTTL;
+    }
+    /**
+     * Store webpage content in cache
+     */
+    cacheContent(url, format, content) {
+        const cacheKey = this.generateCacheKey(url, format);
+        this.contentCache.set(cacheKey, {
+            timestamp: Date.now(),
+            content
+        });
+        // Limit cache size to prevent memory issues (max 50 entries)
+        if (this.contentCache.size > 50) {
+            // Delete oldest entry
+            const oldestKey = Array.from(this.contentCache.entries())
+                .sort((a, b) => a[1].timestamp - b[1].timestamp)[0][0];
+            this.contentCache.delete(oldestKey);
+        }
+    }
+    /**
+     * Generates a concise summary of the content
+     * @param content The content to summarize
+     * @param maxLength Maximum length of the summary
+     * @returns A summary of the content
+     */
+    generateSummary(content, maxLength = 300) {
+        // Simple summarization: take first few sentences up to maxLength
+        const sentences = content.split(/(?<=[.!?])\s+/);
+        let summary = '';
+        for (const sentence of sentences) {
+            if ((summary + sentence).length <= maxLength) {
+                summary += sentence + ' ';
+            }
+            else {
+                break;
+            }
+        }
+        return summary.trim() + (summary.length < content.length ? '...' : '');
+    }
+    async extractContent(url, format = 'markdown') {
+        if (!this.isValidUrl(url)) {
+            throw new Error('Invalid URL provided');
+        }
+        // Check cache first
+        const cacheKey = this.generateCacheKey(url, format);
+        const cachedContent = this.contentCache.get(cacheKey);
+        if (cachedContent && this.isCacheValid(cachedContent)) {
+            console.error(`Using cached content for ${url}`);
+            return cachedContent.content;
+        }
+        try {
+            // Fetch webpage content
+            const response = await axios.get(url, {
+                headers: {
+                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+                },
+                timeout: 10000
+            });
+            // Parse with Cheerio for metadata
+            const $ = cheerio.load(response.data);
+            const metaTags = {};
+            // Only extract the most important meta tags to reduce data volume
+            const importantMetaTags = ['description', 'keywords', 'author', 'og:title', 'og:description', 'twitter:title', 'twitter:description'];
+            $('meta').each((_, element) => {
+                const name = $(element).attr('name') || $(element).attr('property') || '';
+                const content = $(element).attr('content') || '';
+                if (name && content && importantMetaTags.some(tag => name.includes(tag))) {
+                    metaTags[name] = content;
+                }
+            });
+            // Use Readability for main content extraction
+            const dom = new JSDOM(response.data);
+            const reader = new Readability(dom.window.document);
+            const article = reader.parse();
+            if (!article) {
+                throw new Error('Failed to extract content from webpage');
+            }
+            // Convert content based on requested format
+            let contentStr;
+            switch (format) {
+                case 'html':
+                    contentStr = article.content || '';
+                    break;
+                case 'text':
+                    contentStr = this.htmlToPlainText(article.content || '');
+                    break;
+                case 'markdown':
+                default:
+                    contentStr = this.htmlToMarkdown(article.content || '');
+                    break;
+            }
+            // Calculate content stats
+            const wordCount = contentStr.split(/\s+/).filter(word => word.length > 0).length;
+            // Generate a summary of the content
+            const summary = this.generateSummary(contentStr);
+            const content = {
+                url,
+                title: $('title').text() || article.title || '',
+                description: metaTags['description'] || '',
+                content: contentStr,
+                format: format,
+                meta_tags: metaTags,
+                stats: {
+                    word_count: wordCount,
+                    approximate_chars: contentStr.length
+                },
+                content_preview: {
+                    first_500_chars: contentStr.slice(0, 500) + (contentStr.length > 500 ? '...' : '')
+                },
+                summary: summary
+            };
+            // Cache the content before returning
+            this.cacheContent(url, format, content);
+            return content;
+        }
+        catch (error) {
+            if (axios.isAxiosError(error)) {
+                throw new Error(`Failed to fetch webpage: ${error.message}`);
+            }
+            throw error;
+        }
+    }
+    async batchExtractContent(urls, format = 'markdown') {
+        const results = {};
+        await Promise.all(urls.map(async (url) => {
+            try {
+                results[url] = await this.extractContent(url, format);
+            }
+            catch (error) {
+                results[url] = {
+                    error: error instanceof Error ? error.message : 'Unknown error occurred'
+                };
+            }
+        }));
+        return results;
+    }
+}