pse-mcp 0.1.0 → 0.1.2

package/dist/services/google-search.service.js

@@ -1,244 +0,0 @@
-import { google } from 'googleapis';
-import { URL } from 'url';
-export class GoogleSearchService {
-    constructor() {
-        // Cache for search results (key: query string + filters, value: results)
-        this.searchCache = new Map();
-        // Cache expiration time in milliseconds (5 minutes)
-        this.cacheTTL = 5 * 60 * 1000;
-        const apiKey = process.env.GOOGLE_API_KEY;
-        const searchEngineId = process.env.GOOGLE_SEARCH_ENGINE_ID;
-        if (!apiKey || !searchEngineId) {
-            throw new Error('Missing required environment variables: GOOGLE_API_KEY and GOOGLE_SEARCH_ENGINE_ID');
-        }
-        // Initialize Google Custom Search API
-        this.customSearch = google.customsearch('v1').cse;
-        this.searchEngineId = searchEngineId;
-        // Set up the API client
-        google.options({
-            auth: apiKey
-        });
-    }
-    /**
-     * Generate a cache key from search parameters
-     */
-    generateCacheKey(query, numResults, filters) {
-        return JSON.stringify({
-            query,
-            numResults,
-            filters
-        });
-    }
-    /**
-     * Check if a cache entry is still valid
-     */
-    isCacheValid(entry) {
-        const now = Date.now();
-        return now - entry.timestamp < this.cacheTTL;
-    }
-    /**
-     * Store search results in cache
-     */
-    cacheSearchResults(cacheKey, results, pagination, categories) {
-        this.searchCache.set(cacheKey, {
-            timestamp: Date.now(),
-            data: { results, pagination, categories }
-        });
-        // Limit cache size to prevent memory issues (max 100 entries)
-        if (this.searchCache.size > 100) {
-            // Delete oldest entry
-            const oldestKey = Array.from(this.searchCache.entries())
-                .sort((a, b) => a[1].timestamp - b[1].timestamp)[0][0];
-            this.searchCache.delete(oldestKey);
-        }
-    }
-    async search(query, numResults = 5, filters) {
-        try {
-            // Generate cache key
-            const cacheKey = this.generateCacheKey(query, numResults, filters);
-            // Check cache first
-            const cachedResult = this.searchCache.get(cacheKey);
-            if (cachedResult && this.isCacheValid(cachedResult)) {
-                console.error('Using cached search results');
-                return cachedResult.data;
-            }
-            let formattedQuery = query;
-            // Apply site filter if provided
-            if (filters?.site) {
-                formattedQuery += ` site:${filters.site}`;
-            }
-            // Apply exact terms if provided
-            if (filters?.exactTerms) {
-                formattedQuery += ` "${filters.exactTerms}"`;
-            }
-            // Set default pagination values if not provided
-            const page = filters?.page && filters.page > 0 ? filters.page : 1;
-            const resultsPerPage = filters?.resultsPerPage ? Math.min(filters.resultsPerPage, 10) : Math.min(numResults, 10);
-            // Calculate start index for pagination (Google uses 1-based indexing)
-            const startIndex = (page - 1) * resultsPerPage + 1;
-            const params = {
-                cx: this.searchEngineId,
-                q: formattedQuery,
-                num: resultsPerPage,
-                start: startIndex
-            };
-            // Apply language filter if provided
-            if (filters?.language) {
-                params.lr = `lang_${filters.language}`;
-            }
-            // Apply date restriction if provided
-            if (filters?.dateRestrict) {
-                params.dateRestrict = filters.dateRestrict;
-            }
-            // Apply result type filter if provided
-            if (filters?.resultType) {
-                switch (filters.resultType.toLowerCase()) {
-                    case 'image':
-                    case 'images':
-                        params.searchType = 'image';
-                        break;
-                    case 'news':
-                        // For news, we need to modify the query
-                        formattedQuery += ' source:news';
-                        params.q = formattedQuery;
-                        break;
-                    case 'video':
-                    case 'videos':
-                        // For videos, we can use a more specific filter
-                        formattedQuery += ' filetype:video OR inurl:video OR inurl:watch';
-                        params.q = formattedQuery;
-                        break;
-                }
-            }
-            // Apply sorting if provided
-            if (filters?.sort) {
-                switch (filters.sort.toLowerCase()) {
-                    case 'date':
-                        // Sort by date (most recent first)
-                        params.sort = 'date';
-                        break;
-                    case 'relevance':
-                    default:
-                        // Google's default sort is by relevance, so we don't need to specify
-                        break;
-                }
-            }
-            const response = await this.customSearch.list(params);
-            // If no items are found, return empty results with pagination info
-            if (!response.data.items) {
-                return {
-                    results: [],
-                    pagination: {
-                        currentPage: page,
-                        resultsPerPage,
-                        totalResults: 0,
-                        totalPages: 0,
-                        hasNextPage: false,
-                        hasPreviousPage: page > 1
-                    },
-                    categories: []
-                };
-            }
-            // Map the search results and categorize them
-            const results = response.data.items.map(item => {
-                const result = {
-                    title: item.title || '',
-                    link: item.link || '',
-                    snippet: item.snippet || '',
-                    pagemap: item.pagemap || {},
-                    datePublished: item.pagemap?.metatags?.[0]?.['article:published_time'] || '',
-                    source: 'google_search'
-                };
-                // Add category to the result
-                result.category = this.categorizeResult(result);
-                return result;
-            });
-            // Generate category statistics
-            const categories = this.generateCategoryStats(results);
-            // Create pagination information
-            const totalResults = parseInt(response.data.searchInformation?.totalResults || '0', 10);
-            const totalPages = Math.ceil(totalResults / resultsPerPage);
-            const pagination = {
-                currentPage: page,
-                resultsPerPage,
-                totalResults,
-                totalPages,
-                hasNextPage: page < totalPages,
-                hasPreviousPage: page > 1
-            };
-            // Cache the results before returning
-            this.cacheSearchResults(cacheKey, results, pagination, categories);
-            return {
-                results,
-                pagination,
-                categories
-            };
-        }
-        catch (error) {
-            if (error instanceof Error) {
-                throw new Error(`Google Search API error: ${error.message}`);
-            }
-            throw new Error('Unknown error during Google search');
-        }
-    }
-    /**
-     * Categorizes a search result based on its content
-     * @param result The search result to categorize
-     * @returns The category name
-     */
-    categorizeResult(result) {
-        try {
-            // Extract the domain from the URL
-            const url = new URL(result.link);
-            const domain = url.hostname.replace(/^www\./, '');
-            // Check if this is a social media site
-            if (domain.match(/facebook\.com|twitter\.com|instagram\.com|linkedin\.com|pinterest\.com|tiktok\.com|reddit\.com/i)) {
-                return 'Social Media';
-            }
-            // Check if this is a video site
-            if (domain.match(/youtube\.com|vimeo\.com|dailymotion\.com|twitch\.tv/i)) {
-                return 'Video';
-            }
-            // Check if this is a news site
-            if (domain.match(/news|cnn\.com|bbc\.com|nytimes\.com|wsj\.com|reuters\.com|bloomberg\.com/i)) {
-                return 'News';
-            }
-            // Check if this is an educational site
-            if (domain.match(/\.edu$|wikipedia\.org|khan|course|learn|study|academic/i)) {
-                return 'Educational';
-            }
-            // Check if this is a documentation site
-            if (domain.match(/docs|documentation|developer|github\.com|gitlab\.com|bitbucket\.org|stackoverflow\.com/i) ||
-                result.title.match(/docs|documentation|api|reference|manual/i)) {
-                return 'Documentation';
-            }
-            // Check if this is a shopping site
-            if (domain.match(/amazon\.com|ebay\.com|etsy\.com|walmart\.com|shop|store|buy/i)) {
-                return 'Shopping';
-            }
-            // Default category based on domain
-            return domain.split('.').slice(-2, -1)[0].charAt(0).toUpperCase() + domain.split('.').slice(-2, -1)[0].slice(1);
-        }
-        catch (error) {
-            // If there's any error in categorization, return a default category
-            return 'Other';
-        }
-    }
-    /**
-     * Generates category statistics from search results
-     * @param results The search results to analyze
-     * @returns An array of category information
-     */
-    generateCategoryStats(results) {
-        // Count results by category
-        const categoryCounts = {};
-        results.forEach(result => {
-            const category = result.category || 'Other';
-            categoryCounts[category] = (categoryCounts[category] || 0) + 1;
-        });
-        // Convert to array of category info objects
-        return Object.entries(categoryCounts)
-            .map(([name, count]) => ({ name, count }))
-            .sort((a, b) => b.count - a.count); // Sort by count in descending order
-    }
-}

package/dist/types.js

@@ -1 +0,0 @@
-export {};

package/src/services/content-extractor.service.ts

@@ -1,232 +0,0 @@
-import axios from 'axios';
-import * as cheerio from 'cheerio';
-import { Readability } from '@mozilla/readability';
-import { JSDOM } from 'jsdom';
-import MarkdownIt from 'markdown-it';
-import { WebpageContent, OutputFormat } from '../types.js';
-import TurndownService from 'turndown';
-
-interface ContentCacheEntry {
-  timestamp: number;
-  content: WebpageContent;
-}
-
-export class ContentExtractor {
-  private md: MarkdownIt;
-  private turndownService: TurndownService;
-  // Cache for webpage content (key: url + format, value: content)
-  private contentCache: Map<string, ContentCacheEntry> = new Map();
-  // Cache expiration time in milliseconds (30 minutes)
-  private cacheTTL: number = 30 * 60 * 1000;
-
-  constructor() {
-    this.md = new MarkdownIt();
-    this.turndownService = new TurndownService({
-      headingStyle: 'atx',
-      codeBlockStyle: 'fenced'
-    });
-  }
-
-  private cleanText(text: string): string {
-    // Remove multiple blank lines
-    text = text.replace(/\n\s*\n\s*\n/g, '\n\n');
-    // Remove excessive spaces
-    text = text.replace(/ +/g, ' ');
-    return text.trim();
-  }
-
-  private cleanMarkdown(text: string): string {
-    let cleanedText = this.cleanText(text);
-    // Ensure headers have space after #
-    cleanedText = cleanedText.replace(/#([A-Za-z0-9])/g, '# $1');
-    return cleanedText;
-  }
-
-  private htmlToMarkdown(html: string): string {
-    return this.cleanMarkdown(this.turndownService.turndown(html));
-  }
-
-  private htmlToPlainText(html: string): string {
-    const dom = new JSDOM(html);
-    return this.cleanText(dom.window.document.body.textContent || '');
-  }
-
-  private isValidUrl(url: string): boolean {
-    try {
-      new URL(url);
-      return true;
-    } catch {
-      return false;
-    }
-  }
-
-  /**
-   * Generate a cache key from URL and format
-   */
-  private generateCacheKey(url: string, format: OutputFormat): string {
-    return `${url}|${format}`;
-  }
-
-  /**
-   * Check if a cache entry is still valid
-   */
-  private isCacheValid(entry: ContentCacheEntry): boolean {
-    const now = Date.now();
-    return now - entry.timestamp < this.cacheTTL;
-  }
-
-  /**
-   * Store webpage content in cache
-   */
-  private cacheContent(url: string, format: OutputFormat, content: WebpageContent): void {
-    const cacheKey = this.generateCacheKey(url, format);
-    this.contentCache.set(cacheKey, {
-      timestamp: Date.now(),
-      content
-    });
-    
-    // Limit cache size to prevent memory issues (max 50 entries)
-    if (this.contentCache.size > 50) {
-      // Delete oldest entry
-      const oldestKey = Array.from(this.contentCache.entries())
-        .sort((a, b) => a[1].timestamp - b[1].timestamp)[0][0];
-      this.contentCache.delete(oldestKey);
-    }
-  }
-
-  /**
-   * Generates a concise summary of the content
-   * @param content The content to summarize
-   * @param maxLength Maximum length of the summary
-   * @returns A summary of the content
-   */
-  private generateSummary(content: string, maxLength: number = 300): string {
-    // Simple summarization: take first few sentences up to maxLength
-    const sentences = content.split(/(?<=[.!?])\s+/);
-    let summary = '';
-    
-    for (const sentence of sentences) {
-      if ((summary + sentence).length <= maxLength) {
-        summary += sentence + ' ';
-      } else {
-        break;
-      }
-    }
-    
-    return summary.trim() + (summary.length < content.length ? '...' : '');
-  }
-
-  async extractContent(url: string, format: OutputFormat = 'markdown'): Promise<WebpageContent> {
-    if (!this.isValidUrl(url)) {
-      throw new Error('Invalid URL provided');
-    }
-
-    // Check cache first
-    const cacheKey = this.generateCacheKey(url, format);
-    const cachedContent = this.contentCache.get(cacheKey);
-    if (cachedContent && this.isCacheValid(cachedContent)) {
-      console.error(`Using cached content for ${url}`);
-      return cachedContent.content;
-    }
-
-    try {
-      // Fetch webpage content
-      const response = await axios.get(url, {
-        headers: {
-          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
-        },
-        timeout: 10000
-      });
-
-      // Parse with Cheerio for metadata
-      const $ = cheerio.load(response.data);
-      const metaTags: Record<string, string> = {};
-      
-      // Only extract the most important meta tags to reduce data volume
-      const importantMetaTags = ['description', 'keywords', 'author', 'og:title', 'og:description', 'twitter:title', 'twitter:description'];
-      
-      $('meta').each((_, element) => {
-        const name = $(element).attr('name') || $(element).attr('property') || '';
-        const content = $(element).attr('content') || '';
-        if (name && content && importantMetaTags.some(tag => name.includes(tag))) {
-          metaTags[name] = content;
-        }
-      });
-
-      // Use Readability for main content extraction
-      const dom = new JSDOM(response.data);
-      const reader = new Readability(dom.window.document);
-      const article = reader.parse();
-
-      if (!article) {
-        throw new Error('Failed to extract content from webpage');
-      }
-
-      // Convert content based on requested format
-      let contentStr: string;
-      switch (format) {
-        case 'html':
-          contentStr = article.content || '';
-          break;
-        case 'text':
-          contentStr = this.htmlToPlainText(article.content || '');
-          break;
-        case 'markdown':
-        default:
-          contentStr = this.htmlToMarkdown(article.content || '');
-          break;
-      }
-
-      // Calculate content stats
-      const wordCount = contentStr.split(/\s+/).filter(word => word.length > 0).length;
-      
-      // Generate a summary of the content
-      const summary = this.generateSummary(contentStr);
-
-      const content: WebpageContent = {
-        url,
-        title: ($('title').text() as string) || article.title || '',
-        description: metaTags['description'] || '',
-        content: contentStr,
-        format: format,
-        meta_tags: metaTags,
-        stats: {
-          word_count: wordCount,
-          approximate_chars: contentStr.length
-        },
-        content_preview: {
-          first_500_chars: contentStr.slice(0, 500) + (contentStr.length > 500 ? '...' : '')
-        },
-        summary: summary
-      };
-
-      // Cache the content before returning
-      this.cacheContent(url, format, content);
-
-      return content;
-    } catch (error) {
-      if (axios.isAxiosError(error)) {
-        throw new Error(`Failed to fetch webpage: ${error.message}`);
-      }
-      throw error;
-    }
-  }
-
-  async batchExtractContent(urls: string[], format: OutputFormat = 'markdown'): Promise<Record<string, WebpageContent | { error: string }>> {
-    const results: Record<string, WebpageContent | { error: string }> = {};
-
-    await Promise.all(
-      urls.map(async (url) => {
-        try {
-          results[url] = await this.extractContent(url, format);
-        } catch (error) {
-          results[url] = {
-            error: error instanceof Error ? error.message : 'Unknown error occurred'
-          };
-        }
-      })
-    );
-
-    return results;
-  }
-}

package/tasks.md

@@ -1,141 +0,0 @@
-# Google Search MCP Server Improvement Plan
-
-## Current Implementation
-The Google Search MCP Server currently provides three tools:
-1. `google_search` - Searches Google and returns relevant results
-2. `extract_webpage_content` - Extracts content from a single webpage
-3. `extract_multiple_webpages` - Extracts content from multiple webpages
-
-## Improvement Roadmap
-
-### Phase 1: Enhanced Search Capabilities
-
-#### Task 1: Add Basic Search Filters
-- [x] Add site-specific search parameter (`site:example.com`)
-- [x] Add language filter parameter
-- [x] Update input schema and documentation
-- [x] Test functionality with different filter combinations
-
-#### Task 2: Add Date Range Filtering
-- [x] Add date range parameters (start date, end date)
-- [x] Implement date formatting and validation
-- [x] Update Google API query construction
-- [x] Test functionality with various date ranges
-
-#### Task 3: Add Result Type Specification
-- [x] Add parameter for result type (news, images, videos)
-- [x] Implement type-specific query parameters
-- [x] Update result processing for each type
-- [x] Test different result types
-
-#### Task 4: Implement Pagination Support
-- [x] Add pagination parameters (page number, results per page)
-- [x] Implement pagination logic using Google API's `start` parameter
-- [x] Add metadata about total results and current page
-- [x] Test pagination functionality
-
-#### Task 5: Add Sorting Options
-- [x] Add sorting parameter (relevance, date)
-- [x] Implement sort parameter handling
-- [x] Test different sorting options
-
-#### Task 6: Implement Result Categorization
-- [x] Design categorization algorithm (by domain, topic, or content type)
-- [x] Implement result clustering/categorization
-- [x] Add category information to the response
-- [x] Test categorization with various search queries
-
-### Phase 2: Advanced Content Extraction
-
-#### Task 7: Support Different Output Formats
-- [x] Add parameter for output format (markdown, HTML, plain text)
-- [x] Implement format conversion functions
-- [x] Test output in different formats
-
-#### Task 8: Add Content Summarization
-- [ ] Research summarization approaches
-- [ ] Implement text summarization algorithm
-- [ ] Add summary to extraction results
-- [ ] Test summarization with various content types
-
-#### Task 9: Extract Specific Elements
-- [ ] Add support for extracting tables
-- [ ] Add support for extracting lists
-- [ ] Add support for extracting code blocks
-- [ ] Test specific element extraction
-
-#### Task 10: Implement Image Extraction
-- [ ] Add functionality to identify and extract images
-- [ ] Process image metadata (alt text, dimensions)
-- [ ] Return image URLs and metadata
-- [ ] Test image extraction from various pages
-
-#### Task 11: Add Content Translation Support
-- [ ] Research translation API options
-- [ ] Integrate with a translation service
-- [ ] Add target language parameter
-- [ ] Test translation functionality
-
-### Phase 3: Performance and Infrastructure Improvements
-
-#### Task 12: Implement Result Caching
-- [x] Design cache structure for search results
-- [x] Implement cache lookup before making new requests
-- [x] Add cache expiration mechanism
-- [x] Test cache hit and miss scenarios
-
-#### Task 13: Add Content Cache Layer
-- [x] Design cache structure for webpage content
-- [x] Implement content cache lookup and storage
-- [x] Add cache invalidation strategy
-- [x] Test content caching performance
-
-#### Task 14: Implement Rate Limiting
-- [ ] Add rate limiting configuration
-- [ ] Implement request throttling
-- [ ] Add rate limit information in responses
-- [ ] Test rate limiting behavior
-
-#### Task 15: Add Concurrent Request Handling
-- [ ] Implement batch processing for search requests
-- [ ] Add parallel processing for multiple content extractions
-- [ ] Optimize resource usage during concurrent operations
-- [ ] Test performance with concurrent requests
-
-#### Task 16: Support Custom User-Agent Strings
-- [ ] Add user-agent parameter
-- [ ] Implement user-agent validation
-- [ ] Update request headers with custom user-agent
-- [ ] Test different user-agent strings
-
-#### Task 17: Add Proxy Support
-- [ ] Add proxy configuration options
-- [ ] Implement proxy routing for requests
-- [ ] Add fallback mechanism for proxy failures
-- [ ] Test proxy functionality
-
-### Phase 4: Finalization and Documentation
-
-#### Task 18: Comprehensive Testing
-- [ ] Develop automated tests for all new features
-- [ ] Perform integration testing
-- [ ] Stress test with high volume of requests
-- [ ] Fix any identified issues
-
-#### Task 19: Update Documentation
-- [ ] Update server documentation with new capabilities
-- [ ] Create examples for each new feature
-- [ ] Document best practices and recommendations
-- [ ] Create troubleshooting guide
-
-#### Task 20: Performance Optimization
-- [ ] Profile and identify bottlenecks
-- [ ] Optimize resource usage
-- [ ] Implement additional caching if needed
-- [ ] Benchmark performance improvements
-
-## Implementation Notes
-- Each task should be completed and tested independently
-- Regular commits should be made after each feature is implemented
-- Follow existing code patterns and naming conventions
-- Maintain backward compatibility where possible