pse-mcp 0.1.0 → 0.1.1

package/dist/content-fetcher.js

@@ -1,36 +0,0 @@
-import axios from 'axios';
-export class ContentFetcher {
-    constructor(port = 5001) {
-        this.baseUrl = `http://localhost:${port}`;
-    }
-    async fetchContent(url) {
-        try {
-            const response = await axios.post(`${this.baseUrl}/analyze`, { url });
-            return response.data;
-        }
-        catch (error) {
-            if (axios.isAxiosError(error)) {
-                throw new Error(`Failed to fetch content: ${error.response?.data?.error || error.message}`);
-            }
-            if (error instanceof Error) {
-                throw new Error(`Failed to fetch content: ${error.message}`);
-            }
-            throw new Error('Failed to fetch content: Unknown error');
-        }
-    }
-    async batchFetchContent(urls) {
-        try {
-            const response = await axios.post(`${this.baseUrl}/batch_analyze`, { urls });
-            return response.data;
-        }
-        catch (error) {
-            if (axios.isAxiosError(error)) {
-                throw new Error(`Failed to batch fetch content: ${error.response?.data?.error || error.message}`);
-            }
-            if (error instanceof Error) {
-                throw new Error(`Failed to batch fetch content: ${error.message}`);
-            }
-            throw new Error('Failed to batch fetch content: Unknown error');
-        }
-    }
-}

package/dist/services/content-extractor.service.js

@@ -1,195 +0,0 @@
-import axios from 'axios';
-import * as cheerio from 'cheerio';
-import { Readability } from '@mozilla/readability';
-import { JSDOM } from 'jsdom';
-import MarkdownIt from 'markdown-it';
-import TurndownService from 'turndown';
-export class ContentExtractor {
-    constructor() {
-        // Cache for webpage content (key: url + format, value: content)
-        this.contentCache = new Map();
-        // Cache expiration time in milliseconds (30 minutes)
-        this.cacheTTL = 30 * 60 * 1000;
-        this.md = new MarkdownIt();
-        this.turndownService = new TurndownService({
-            headingStyle: 'atx',
-            codeBlockStyle: 'fenced'
-        });
-    }
-    cleanText(text) {
-        // Remove multiple blank lines
-        text = text.replace(/\n\s*\n\s*\n/g, '\n\n');
-        // Remove excessive spaces
-        text = text.replace(/ +/g, ' ');
-        return text.trim();
-    }
-    cleanMarkdown(text) {
-        let cleanedText = this.cleanText(text);
-        // Ensure headers have space after #
-        cleanedText = cleanedText.replace(/#([A-Za-z0-9])/g, '# $1');
-        return cleanedText;
-    }
-    htmlToMarkdown(html) {
-        return this.cleanMarkdown(this.turndownService.turndown(html));
-    }
-    htmlToPlainText(html) {
-        const dom = new JSDOM(html);
-        return this.cleanText(dom.window.document.body.textContent || '');
-    }
-    isValidUrl(url) {
-        try {
-            new URL(url);
-            return true;
-        }
-        catch {
-            return false;
-        }
-    }
-    /**
-     * Generate a cache key from URL and format
-     */
-    generateCacheKey(url, format) {
-        return `${url}|${format}`;
-    }
-    /**
-     * Check if a cache entry is still valid
-     */
-    isCacheValid(entry) {
-        const now = Date.now();
-        return now - entry.timestamp < this.cacheTTL;
-    }
-    /**
-     * Store webpage content in cache
-     */
-    cacheContent(url, format, content) {
-        const cacheKey = this.generateCacheKey(url, format);
-        this.contentCache.set(cacheKey, {
-            timestamp: Date.now(),
-            content
-        });
-        // Limit cache size to prevent memory issues (max 50 entries)
-        if (this.contentCache.size > 50) {
-            // Delete oldest entry
-            const oldestKey = Array.from(this.contentCache.entries())
-                .sort((a, b) => a[1].timestamp - b[1].timestamp)[0][0];
-            this.contentCache.delete(oldestKey);
-        }
-    }
-    /**
-     * Generates a concise summary of the content
-     * @param content The content to summarize
-     * @param maxLength Maximum length of the summary
-     * @returns A summary of the content
-     */
-    generateSummary(content, maxLength = 300) {
-        // Simple summarization: take first few sentences up to maxLength
-        const sentences = content.split(/(?<=[.!?])\s+/);
-        let summary = '';
-        for (const sentence of sentences) {
-            if ((summary + sentence).length <= maxLength) {
-                summary += sentence + ' ';
-            }
-            else {
-                break;
-            }
-        }
-        return summary.trim() + (summary.length < content.length ? '...' : '');
-    }
-    async extractContent(url, format = 'markdown') {
-        if (!this.isValidUrl(url)) {
-            throw new Error('Invalid URL provided');
-        }
-        // Check cache first
-        const cacheKey = this.generateCacheKey(url, format);
-        const cachedContent = this.contentCache.get(cacheKey);
-        if (cachedContent && this.isCacheValid(cachedContent)) {
-            console.error(`Using cached content for ${url}`);
-            return cachedContent.content;
-        }
-        try {
-            // Fetch webpage content
-            const response = await axios.get(url, {
-                headers: {
-                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
-                },
-                timeout: 10000
-            });
-            // Parse with Cheerio for metadata
-            const $ = cheerio.load(response.data);
-            const metaTags = {};
-            // Only extract the most important meta tags to reduce data volume
-            const importantMetaTags = ['description', 'keywords', 'author', 'og:title', 'og:description', 'twitter:title', 'twitter:description'];
-            $('meta').each((_, element) => {
-                const name = $(element).attr('name') || $(element).attr('property') || '';
-                const content = $(element).attr('content') || '';
-                if (name && content && importantMetaTags.some(tag => name.includes(tag))) {
-                    metaTags[name] = content;
-                }
-            });
-            // Use Readability for main content extraction
-            const dom = new JSDOM(response.data);
-            const reader = new Readability(dom.window.document);
-            const article = reader.parse();
-            if (!article) {
-                throw new Error('Failed to extract content from webpage');
-            }
-            // Convert content based on requested format
-            let contentStr;
-            switch (format) {
-                case 'html':
-                    contentStr = article.content || '';
-                    break;
-                case 'text':
-                    contentStr = this.htmlToPlainText(article.content || '');
-                    break;
-                case 'markdown':
-                default:
-                    contentStr = this.htmlToMarkdown(article.content || '');
-                    break;
-            }
-            // Calculate content stats
-            const wordCount = contentStr.split(/\s+/).filter(word => word.length > 0).length;
-            // Generate a summary of the content
-            const summary = this.generateSummary(contentStr);
-            const content = {
-                url,
-                title: $('title').text() || article.title || '',
-                description: metaTags['description'] || '',
-                content: contentStr,
-                format: format,
-                meta_tags: metaTags,
-                stats: {
-                    word_count: wordCount,
-                    approximate_chars: contentStr.length
-                },
-                content_preview: {
-                    first_500_chars: contentStr.slice(0, 500) + (contentStr.length > 500 ? '...' : '')
-                },
-                summary: summary
-            };
-            // Cache the content before returning
-            this.cacheContent(url, format, content);
-            return content;
-        }
-        catch (error) {
-            if (axios.isAxiosError(error)) {
-                throw new Error(`Failed to fetch webpage: ${error.message}`);
-            }
-            throw error;
-        }
-    }
-    async batchExtractContent(urls, format = 'markdown') {
-        const results = {};
-        await Promise.all(urls.map(async (url) => {
-            try {
-                results[url] = await this.extractContent(url, format);
-            }
-            catch (error) {
-                results[url] = {
-                    error: error instanceof Error ? error.message : 'Unknown error occurred'
-                };
-            }
-        }));
-        return results;
-    }
-}

package/src/services/content-extractor.service.ts

@@ -1,232 +0,0 @@
-import axios from 'axios';
-import * as cheerio from 'cheerio';
-import { Readability } from '@mozilla/readability';
-import { JSDOM } from 'jsdom';
-import MarkdownIt from 'markdown-it';
-import { WebpageContent, OutputFormat } from '../types.js';
-import TurndownService from 'turndown';
-
-interface ContentCacheEntry {
-  timestamp: number;
-  content: WebpageContent;
-}
-
-export class ContentExtractor {
-  private md: MarkdownIt;
-  private turndownService: TurndownService;
-  // Cache for webpage content (key: url + format, value: content)
-  private contentCache: Map<string, ContentCacheEntry> = new Map();
-  // Cache expiration time in milliseconds (30 minutes)
-  private cacheTTL: number = 30 * 60 * 1000;
-
-  constructor() {
-    this.md = new MarkdownIt();
-    this.turndownService = new TurndownService({
-      headingStyle: 'atx',
-      codeBlockStyle: 'fenced'
-    });
-  }
-
-  private cleanText(text: string): string {
-    // Remove multiple blank lines
-    text = text.replace(/\n\s*\n\s*\n/g, '\n\n');
-    // Remove excessive spaces
-    text = text.replace(/ +/g, ' ');
-    return text.trim();
-  }
-
-  private cleanMarkdown(text: string): string {
-    let cleanedText = this.cleanText(text);
-    // Ensure headers have space after #
-    cleanedText = cleanedText.replace(/#([A-Za-z0-9])/g, '# $1');
-    return cleanedText;
-  }
-
-  private htmlToMarkdown(html: string): string {
-    return this.cleanMarkdown(this.turndownService.turndown(html));
-  }
-
-  private htmlToPlainText(html: string): string {
-    const dom = new JSDOM(html);
-    return this.cleanText(dom.window.document.body.textContent || '');
-  }
-
-  private isValidUrl(url: string): boolean {
-    try {
-      new URL(url);
-      return true;
-    } catch {
-      return false;
-    }
-  }
-
-  /**
-   * Generate a cache key from URL and format
-   */
-  private generateCacheKey(url: string, format: OutputFormat): string {
-    return `${url}|${format}`;
-  }
-
-  /**
-   * Check if a cache entry is still valid
-   */
-  private isCacheValid(entry: ContentCacheEntry): boolean {
-    const now = Date.now();
-    return now - entry.timestamp < this.cacheTTL;
-  }
-
-  /**
-   * Store webpage content in cache
-   */
-  private cacheContent(url: string, format: OutputFormat, content: WebpageContent): void {
-    const cacheKey = this.generateCacheKey(url, format);
-    this.contentCache.set(cacheKey, {
-      timestamp: Date.now(),
-      content
-    });
-    
-    // Limit cache size to prevent memory issues (max 50 entries)
-    if (this.contentCache.size > 50) {
-      // Delete oldest entry
-      const oldestKey = Array.from(this.contentCache.entries())
-        .sort((a, b) => a[1].timestamp - b[1].timestamp)[0][0];
-      this.contentCache.delete(oldestKey);
-    }
-  }
-
-  /**
-   * Generates a concise summary of the content
-   * @param content The content to summarize
-   * @param maxLength Maximum length of the summary
-   * @returns A summary of the content
-   */
-  private generateSummary(content: string, maxLength: number = 300): string {
-    // Simple summarization: take first few sentences up to maxLength
-    const sentences = content.split(/(?<=[.!?])\s+/);
-    let summary = '';
-    
-    for (const sentence of sentences) {
-      if ((summary + sentence).length <= maxLength) {
-        summary += sentence + ' ';
-      } else {
-        break;
-      }
-    }
-    
-    return summary.trim() + (summary.length < content.length ? '...' : '');
-  }
-
-  async extractContent(url: string, format: OutputFormat = 'markdown'): Promise<WebpageContent> {
-    if (!this.isValidUrl(url)) {
-      throw new Error('Invalid URL provided');
-    }
-
-    // Check cache first
-    const cacheKey = this.generateCacheKey(url, format);
-    const cachedContent = this.contentCache.get(cacheKey);
-    if (cachedContent && this.isCacheValid(cachedContent)) {
-      console.error(`Using cached content for ${url}`);
-      return cachedContent.content;
-    }
-
-    try {
-      // Fetch webpage content
-      const response = await axios.get(url, {
-        headers: {
-          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
-        },
-        timeout: 10000
-      });
-
-      // Parse with Cheerio for metadata
-      const $ = cheerio.load(response.data);
-      const metaTags: Record<string, string> = {};
-      
-      // Only extract the most important meta tags to reduce data volume
-      const importantMetaTags = ['description', 'keywords', 'author', 'og:title', 'og:description', 'twitter:title', 'twitter:description'];
-      
-      $('meta').each((_, element) => {
-        const name = $(element).attr('name') || $(element).attr('property') || '';
-        const content = $(element).attr('content') || '';
-        if (name && content && importantMetaTags.some(tag => name.includes(tag))) {
-          metaTags[name] = content;
-        }
-      });
-
-      // Use Readability for main content extraction
-      const dom = new JSDOM(response.data);
-      const reader = new Readability(dom.window.document);
-      const article = reader.parse();
-
-      if (!article) {
-        throw new Error('Failed to extract content from webpage');
-      }
-
-      // Convert content based on requested format
-      let contentStr: string;
-      switch (format) {
-        case 'html':
-          contentStr = article.content || '';
-          break;
-        case 'text':
-          contentStr = this.htmlToPlainText(article.content || '');
-          break;
-        case 'markdown':
-        default:
-          contentStr = this.htmlToMarkdown(article.content || '');
-          break;
-      }
-
-      // Calculate content stats
-      const wordCount = contentStr.split(/\s+/).filter(word => word.length > 0).length;
-      
-      // Generate a summary of the content
-      const summary = this.generateSummary(contentStr);
-
-      const content: WebpageContent = {
-        url,
-        title: ($('title').text() as string) || article.title || '',
-        description: metaTags['description'] || '',
-        content: contentStr,
-        format: format,
-        meta_tags: metaTags,
-        stats: {
-          word_count: wordCount,
-          approximate_chars: contentStr.length
-        },
-        content_preview: {
-          first_500_chars: contentStr.slice(0, 500) + (contentStr.length > 500 ? '...' : '')
-        },
-        summary: summary
-      };
-
-      // Cache the content before returning
-      this.cacheContent(url, format, content);
-
-      return content;
-    } catch (error) {
-      if (axios.isAxiosError(error)) {
-        throw new Error(`Failed to fetch webpage: ${error.message}`);
-      }
-      throw error;
-    }
-  }
-
-  async batchExtractContent(urls: string[], format: OutputFormat = 'markdown'): Promise<Record<string, WebpageContent | { error: string }>> {
-    const results: Record<string, WebpageContent | { error: string }> = {};
-
-    await Promise.all(
-      urls.map(async (url) => {
-        try {
-          results[url] = await this.extractContent(url, format);
-        } catch (error) {
-          results[url] = {
-            error: error instanceof Error ? error.message : 'Unknown error occurred'
-          };
-        }
-      })
-    );
-
-    return results;
-  }
-}

package/tasks.md

@@ -1,141 +0,0 @@
-# Google Search MCP Server Improvement Plan
-
-## Current Implementation
-The Google Search MCP Server currently provides three tools:
-1. `google_search` - Searches Google and returns relevant results
-2. `extract_webpage_content` - Extracts content from a single webpage
-3. `extract_multiple_webpages` - Extracts content from multiple webpages
-
-## Improvement Roadmap
-
-### Phase 1: Enhanced Search Capabilities
-
-#### Task 1: Add Basic Search Filters
-- [x] Add site-specific search parameter (`site:example.com`)
-- [x] Add language filter parameter
-- [x] Update input schema and documentation
-- [x] Test functionality with different filter combinations
-
-#### Task 2: Add Date Range Filtering
-- [x] Add date range parameters (start date, end date)
-- [x] Implement date formatting and validation
-- [x] Update Google API query construction
-- [x] Test functionality with various date ranges
-
-#### Task 3: Add Result Type Specification
-- [x] Add parameter for result type (news, images, videos)
-- [x] Implement type-specific query parameters
-- [x] Update result processing for each type
-- [x] Test different result types
-
-#### Task 4: Implement Pagination Support
-- [x] Add pagination parameters (page number, results per page)
-- [x] Implement pagination logic using Google API's `start` parameter
-- [x] Add metadata about total results and current page
-- [x] Test pagination functionality
-
-#### Task 5: Add Sorting Options
-- [x] Add sorting parameter (relevance, date)
-- [x] Implement sort parameter handling
-- [x] Test different sorting options
-
-#### Task 6: Implement Result Categorization
-- [x] Design categorization algorithm (by domain, topic, or content type)
-- [x] Implement result clustering/categorization
-- [x] Add category information to the response
-- [x] Test categorization with various search queries
-
-### Phase 2: Advanced Content Extraction
-
-#### Task 7: Support Different Output Formats
-- [x] Add parameter for output format (markdown, HTML, plain text)
-- [x] Implement format conversion functions
-- [x] Test output in different formats
-
-#### Task 8: Add Content Summarization
-- [ ] Research summarization approaches
-- [ ] Implement text summarization algorithm
-- [ ] Add summary to extraction results
-- [ ] Test summarization with various content types
-
-#### Task 9: Extract Specific Elements
-- [ ] Add support for extracting tables
-- [ ] Add support for extracting lists
-- [ ] Add support for extracting code blocks
-- [ ] Test specific element extraction
-
-#### Task 10: Implement Image Extraction
-- [ ] Add functionality to identify and extract images
-- [ ] Process image metadata (alt text, dimensions)
-- [ ] Return image URLs and metadata
-- [ ] Test image extraction from various pages
-
-#### Task 11: Add Content Translation Support
-- [ ] Research translation API options
-- [ ] Integrate with a translation service
-- [ ] Add target language parameter
-- [ ] Test translation functionality
-
-### Phase 3: Performance and Infrastructure Improvements
-
-#### Task 12: Implement Result Caching
-- [x] Design cache structure for search results
-- [x] Implement cache lookup before making new requests
-- [x] Add cache expiration mechanism
-- [x] Test cache hit and miss scenarios
-
-#### Task 13: Add Content Cache Layer
-- [x] Design cache structure for webpage content
-- [x] Implement content cache lookup and storage
-- [x] Add cache invalidation strategy
-- [x] Test content caching performance
-
-#### Task 14: Implement Rate Limiting
-- [ ] Add rate limiting configuration
-- [ ] Implement request throttling
-- [ ] Add rate limit information in responses
-- [ ] Test rate limiting behavior
-
-#### Task 15: Add Concurrent Request Handling
-- [ ] Implement batch processing for search requests
-- [ ] Add parallel processing for multiple content extractions
-- [ ] Optimize resource usage during concurrent operations
-- [ ] Test performance with concurrent requests
-
-#### Task 16: Support Custom User-Agent Strings
-- [ ] Add user-agent parameter
-- [ ] Implement user-agent validation
-- [ ] Update request headers with custom user-agent
-- [ ] Test different user-agent strings
-
-#### Task 17: Add Proxy Support
-- [ ] Add proxy configuration options
-- [ ] Implement proxy routing for requests
-- [ ] Add fallback mechanism for proxy failures
-- [ ] Test proxy functionality
-
-### Phase 4: Finalization and Documentation
-
-#### Task 18: Comprehensive Testing
-- [ ] Develop automated tests for all new features
-- [ ] Perform integration testing
-- [ ] Stress test with high volume of requests
-- [ ] Fix any identified issues
-
-#### Task 19: Update Documentation
-- [ ] Update server documentation with new capabilities
-- [ ] Create examples for each new feature
-- [ ] Document best practices and recommendations
-- [ ] Create troubleshooting guide
-
-#### Task 20: Performance Optimization
-- [ ] Profile and identify bottlenecks
-- [ ] Optimize resource usage
-- [ ] Implement additional caching if needed
-- [ ] Benchmark performance improvements
-
-## Implementation Notes
-- Each task should be completed and tested independently
-- Regular commits should be made after each feature is implemented
-- Follow existing code patterns and naming conventions
-- Maintain backward compatibility where possible