pse-mcp 0.1.0 → 0.1.1

package/package.json

@@ -1,40 +1,40 @@
-{
-  "name": "pse-mcp",
-  "version": "0.1.0",
-  "description": "MCP server for Google search and webpage analysis",
-  "type": "module",
-  "bin": {
-    "pse-mcp": "dist/google-search.js"
-  },
-  "scripts": {
-    "build": "tsc",
-    "start": "node dist/google-search.js",
-    "dev": "tsc -w",
-    "start:python": "concurrently \"python google_search.py\" \"python link_view.py\"",
-    "start:all": "concurrently \"npm run start:python\" \"npm run start\""
-  },
-  "dependencies": {
-    "@modelcontextprotocol/sdk": "^1.0.1",
-    "@mozilla/readability": "^0.6.0",
-    "@types/turndown": "^5.0.5",
-    "axios": "^1.7.9",
-    "cheerio": "^1.0.0",
-    "dompurify": "^3.2.3",
-    "express": "^4.21.2",
-    "googleapis": "^144.0.0",
-    "jsdom": "^25.0.1",
-    "markdown-it": "^14.1.0",
-    "readability": "^0.1.0",
-    "turndown": "^7.2.0"
-  },
-  "devDependencies": {
-    "@types/cheerio": "^0.22.35",
-    "@types/dompurify": "^3.0.5",
-    "@types/express": "^4.17.21",
-    "@types/jsdom": "^21.1.7",
-    "@types/markdown-it": "^14.1.2",
-    "@types/node": "^20.17.21",
-    "concurrently": "^9.1.0",
-    "typescript": "^5.7.2"
-  }
-}
+{
+  "name": "pse-mcp",
+  "version": "0.1.1",
+  "description": "MCP server for Google search and webpage analysis",
+  "type": "module",
+  "bin": {
+    "pse-mcp": "dist-package/dist/google-search.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "start": "node dist/google-search.js",
+    "dev": "tsc -w",
+    "start:python": "concurrently \"python google_search.py\" \"python link_view.py\"",
+    "start:all": "concurrently \"npm run start:python\" \"npm run start\""
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.0.1",
+    "@mozilla/readability": "^0.6.0",
+    "@types/turndown": "^5.0.5",
+    "axios": "^1.7.9",
+    "cheerio": "^1.0.0",
+    "dompurify": "^3.2.3",
+    "express": "^4.21.2",
+    "googleapis": "^144.0.0",
+    "jsdom": "^25.0.1",
+    "markdown-it": "^14.1.0",
+    "readability": "^0.1.0",
+    "turndown": "^7.2.0"
+  },
+  "devDependencies": {
+    "@types/cheerio": "^0.22.35",
+    "@types/dompurify": "^3.0.5",
+    "@types/express": "^4.17.21",
+    "@types/jsdom": "^21.1.7",
+    "@types/markdown-it": "^14.1.2",
+    "@types/node": "^20.17.21",
+    "concurrently": "^9.1.0",
+    "typescript": "^5.9.3"
+  }
+}

package/src/google-search.ts

@@ -4,17 +4,13 @@ import { Server } from '@modelcontextprotocol/sdk/server/index.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { CallToolRequestSchema, ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
 import { GoogleSearchService } from './services/google-search.service.js';
-import { ContentExtractor } from './services/content-extractor.service.js';
-import { OutputFormat } from './types.js';
 
 class GoogleSearchServer {
   private server: Server;
   private searchService: GoogleSearchService;
-  private contentExtractor: ContentExtractor;
 
   constructor() {
     this.searchService = new GoogleSearchService();
-    this.contentExtractor = new ContentExtractor();
     this.server = new Server(
       {
         name: 'google-search',
@@ -34,7 +30,7 @@ class GoogleSearchServer {
                 },
                 num_results: { 
                   type: 'number', 
-                  description: 'Number of results to return (default: 5, max: 10). Increase for broader coverage, decrease for faster response.'
+                  description: 'Number of results to return (default: 10, max: 10). Increase for broader coverage, decrease for faster response.'
                 },
                 site: {
                   type: 'string',
@@ -62,7 +58,7 @@ class GoogleSearchServer {
                 },
                 resultsPerPage: {
                   type: 'number',
-                  description: 'Number of results to show per page (default: 5, max: 10). Controls how many results are returned for each page.'
+                  description: 'Number of results to show per page (default: 10, max: 10). Controls how many results are returned for each page.'
                 },
                 sort: {
                   type: 'string',
@@ -71,41 +67,6 @@ class GoogleSearchServer {
               },
               required: ['query']
             }
-          },
-          extract_webpage_content: {
-            description: 'Extract and analyze content from a webpage, converting it to readable text. This tool fetches the main content while removing ads, navigation elements, and other clutter. Use it to get detailed information from specific pages found via google_search. Works with most common webpage formats including articles, blogs, and documentation.',
-            inputSchema: {
-              type: 'object',
-              properties: {
-                url: { 
-                  type: 'string', 
-                  description: 'Full URL of the webpage to extract content from (must start with http:// or https://). Ensure the URL is from a public webpage and not behind authentication.'
-                },
-                format: {
-                  type: 'string',
-                  description: 'Output format for the extracted content. Options: "markdown" (default), "html", or "text".'
-                }
-              },
-              required: ['url']
-            }
-          },
-          extract_multiple_webpages: {
-            description: 'Extract and analyze content from multiple webpages in a single request. This tool is ideal for comparing information across different sources or gathering comprehensive information on a topic. Limited to 5 URLs per request to maintain performance.',
-            inputSchema: {
-              type: 'object',
-              properties: {
-                urls: { 
-                  type: 'array',
-                  items: { type: 'string' },
-                  description: 'Array of webpage URLs to extract content from. Each URL must be public and start with http:// or https://. Maximum 5 URLs per request.'
-                },
-                format: {
-                  type: 'string',
-                  description: 'Output format for the extracted content. Options: "markdown" (default), "html", or "text".'
-                }
-              },
-              required: ['urls']
-            }
           }
         }
       }
@@ -126,7 +87,7 @@ class GoogleSearchServer {
               },
               num_results: { 
                 type: 'number', 
-                description: 'Number of results to return (default: 5, max: 10). Increase for broader coverage, decrease for faster response.'
+                description: 'Number of results to return (default: 10, max: 10). Increase for broader coverage, decrease for faster response.'
               },
               site: {
                 type: 'string',
@@ -154,7 +115,7 @@ class GoogleSearchServer {
               },
               resultsPerPage: {
                 type: 'number',
-                description: 'Number of results to show per page (default: 5, max: 10). Controls how many results are returned for each page.'
+                description: 'Number of results to show per page (default: 10, max: 10). Controls how many results are returned for each page.'
               },
               sort: {
                 type: 'string',
@@ -163,43 +124,6 @@ class GoogleSearchServer {
             },
             required: ['query']
           }
-        },
-        {
-          name: 'extract_webpage_content',
-          description: 'Extract and analyze content from a webpage, converting it to readable text. This tool fetches the main content while removing ads, navigation elements, and other clutter. Use it to get detailed information from specific pages found via google_search. Works with most common webpage formats including articles, blogs, and documentation.',
-          inputSchema: {
-            type: 'object',
-            properties: {
-              url: { 
-                type: 'string', 
-                description: 'Full URL of the webpage to extract content from (must start with http:// or https://). Ensure the URL is from a public webpage and not behind authentication.'
-              },
-              format: {
-                type: 'string',
-                description: 'Output format for the extracted content. Options: "markdown" (default), "html", or "text".'
-              }
-            },
-            required: ['url']
-          }
-        },
-        {
-          name: 'extract_multiple_webpages',
-          description: 'Extract and analyze content from multiple webpages in a single request. This tool is ideal for comparing information across different sources or gathering comprehensive information on a topic. Limited to 5 URLs per request to maintain performance.',
-          inputSchema: {
-            type: 'object',
-            properties: {
-              urls: {
-                type: 'array',
-                items: { type: 'string' },
-                description: 'Array of webpage URLs to extract content from. Each URL must be public and start with http:// or https://. Maximum 5 URLs per request.'
-              },
-              format: {
-                type: 'string',
-                description: 'Output format for the extracted content. Options: "markdown" (default), "html", or "text".'
-              }
-            },
-            required: ['urls']
-          }
         }
       ]
     }));
@@ -226,24 +150,6 @@ class GoogleSearchServer {
           }
           throw new Error('Invalid arguments for google_search tool');
 
-        case 'extract_webpage_content':
-          if (typeof request.params.arguments === 'object' && request.params.arguments !== null && 'url' in request.params.arguments) {
-            return this.handleAnalyzeWebpage({
-              url: String(request.params.arguments.url),
-              format: request.params.arguments.format ? String(request.params.arguments.format) as OutputFormat : 'markdown'
-            });
-          }
-          throw new Error('Invalid arguments for extract_webpage_content tool');
-
-        case 'extract_multiple_webpages':
-          if (typeof request.params.arguments === 'object' && request.params.arguments !== null && 'urls' in request.params.arguments && Array.isArray(request.params.arguments.urls)) {
-            return this.handleBatchAnalyzeWebpages({
-              urls: request.params.arguments.urls.map(String),
-              format: request.params.arguments.format ? String(request.params.arguments.format) as OutputFormat : 'markdown'
-            });
-          }
-          throw new Error('Invalid arguments for extract_multiple_webpages tool');
-
         default:
           throw new Error(`Unknown tool: ${request.params.name}`);
       }
@@ -334,122 +240,6 @@ class GoogleSearchServer {
     }
   }
 
-  private async handleAnalyzeWebpage(args: { url: string; format?: OutputFormat; summarize?: boolean }) {
-    try {
-      const content = await this.contentExtractor.extractContent(args.url, args.format);
-      
-      // Format the response in a more readable, concise way
-      let responseText = `Content from: ${content.url}\n\n`;
-      responseText += `Title: ${content.title}\n`;
-      
-      if (content.description) {
-        responseText += `Description: ${content.description}\n`;
-      }
-      
-      responseText += `\nStats: ${content.stats.word_count} words, ${content.stats.approximate_chars} characters\n\n`;
-      
-      // Add the summary if available
-      if (content.summary) {
-        responseText += `Summary: ${content.summary}\n\n`;
-      }
-      
-      // Add a preview of the content
-      responseText += `Content Preview:\n${content.content_preview.first_500_chars}\n\n`;
-      
-      // Add a note about requesting specific information
-      responseText += `Note: This is a preview of the content. For specific information, please ask about particular aspects of this webpage.`;
-      
-      return {
-        content: [
-          {
-            type: 'text',
-            text: responseText,
-          },
-        ],
-      };
-    } catch (error) {
-      const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred';
-      const helpText = 'Common issues:\n- Check if the URL is accessible in a browser\n- Ensure the webpage is public\n- Try again if it\'s a temporary network issue';
-      
-      return {
-        content: [
-          {
-            type: 'text',
-            text: `${errorMessage}\n\n${helpText}`,
-          },
-        ],
-        isError: true,
-      };
-    }
-  }
-
-  private async handleBatchAnalyzeWebpages(args: { urls: string[]; format?: OutputFormat }) {
-    if (args.urls.length > 5) {
-      return {
-        content: [{ 
-          type: 'text', 
-          text: 'Maximum 5 URLs allowed per request to maintain performance. Please reduce the number of URLs.'
-        }],
-        isError: true
-      };
-    }
-
-    try {
-      const results = await this.contentExtractor.batchExtractContent(args.urls, args.format);
-      
-      // Format the response in a more readable, concise way
-      let responseText = `Content from ${args.urls.length} webpages:\n\n`;
-      
-      for (const [url, result] of Object.entries(results)) {
-        responseText += `URL: ${url}\n`;
-        
-        if ('error' in result) {
-          responseText += `Error: ${result.error}\n\n`;
-          continue;
-        }
-        
-        responseText += `Title: ${result.title}\n`;
-        
-        if (result.description) {
-          responseText += `Description: ${result.description}\n`;
-        }
-        
-        responseText += `Stats: ${result.stats.word_count} words\n`;
-        
-        // Add summary if available
-        if (result.summary) {
-          responseText += `Summary: ${result.summary}\n`;
-        }
-        
-        responseText += `Preview: ${result.content_preview.first_500_chars.substring(0, 150)}...\n\n`;
-      }
-      
-      responseText += `Note: These are previews of the content. To analyze the full content of a specific URL, use the extract_webpage_content tool with that URL.`;
-      
-      return {
-        content: [
-          {
-            type: 'text',
-            text: responseText,
-          },
-        ],
-      };
-    } catch (error) {
-      const errorMessage = error instanceof Error ? error.message : 'Unknown error occurred';
-      const helpText = 'Common issues:\n- Check if all URLs are accessible in a browser\n- Ensure all webpages are public\n- Try again if it\'s a temporary network issue\n- Consider reducing the number of URLs';
-      
-      return {
-        content: [
-          {
-            type: 'text',
-            text: `${errorMessage}\n\n${helpText}`,
-          },
-        ],
-        isError: true,
-      };
-    }
-  }
-
   async start() {
     try {
       const transport = new StdioServerTransport();

package/src/services/google-search.service.ts

@@ -79,7 +79,7 @@ export class GoogleSearchService {
     }
   }
 
-  async search(query: string, numResults: number = 5, filters?: SearchFilters): Promise<{ 
+  async search(query: string, numResults: number = 10, filters?: SearchFilters): Promise<{
     results: SearchResult[]; 
     pagination?: SearchPaginationInfo;
     categories?: CategoryInfo[];

package/src/types.ts

@@ -40,25 +40,3 @@ export interface SearchResponse {
   categories?: CategoryInfo[];
 }
 
-export type OutputFormat = 'markdown' | 'html' | 'text';
-
-export interface WebpageContent {
-  url: string;
-  title: string;
-  description: string;
-  content: string;
-  format: OutputFormat;
-  meta_tags: Record<string, string>;
-  stats: {
-    word_count: number;
-    approximate_chars: number;
-  };
-  content_preview: {
-    first_500_chars: string;
-  };
-  summary?: string;
-}
-
-export interface WebpageAnalysisResponse {
-  [url: string]: WebpageContent | { error: string };
-}

package/GEMINI.md

@@ -1,72 +0,0 @@
-# GEMINI.md
-
-## Project Overview
-
-This project is a TypeScript-based MCP (Model Context Protocol) server that provides Google search capabilities and webpage content analysis tools. It enables AI models to programmatically perform Google searches and analyze webpage content.
-
-The server is built with TypeScript and utilizes the `@modelcontextprotocol/sdk` for MCP communication. It exposes three main tools: `google_search`, `extract_webpage_content`, and `extract_multiple_webpages`.
-
-The core logic is divided into two services:
--   `GoogleSearchService`: Handles interactions with the Google Custom Search API, including caching, pagination, and result categorization.
--   `ContentExtractor`: Fetches and extracts the main content from webpages using `axios`, `cheerio`, and `@mozilla/readability`. It supports various output formats and caching.
-
-## Building and Running
-
-### Prerequisites
-
--   Node.js (v16 or higher)
--   Google Cloud Platform account
--   Custom Search Engine ID
--   Google API Key
-
-### Installation
-
-1.  Install Node.js dependencies:
-    ```bash
-    npm install
-    ```
-
-### Environment Variables
-
-The following environment variables must be set:
-
--   `GOOGLE_API_KEY`: Your Google API key.
--   `GOOGLE_SEARCH_ENGINE_ID`: Your Custom Search Engine ID.
-
-### Building
-
-To build the TypeScript code, run:
-
-```bash
-npm run build
-```
-
-This will compile the TypeScript files from `src` into JavaScript files in the `dist` directory.
-
-### Running
-
-To start the MCP server, run:
-
-```bash
-npm run start
-```
-
-This will execute the compiled `dist/google-search.js` file.
-
-For development, you can use:
-
-```bash
-npm run dev
-```
-
-This will watch for changes in the `src` directory and automatically recompile the code.
-
-## Development Conventions
-
--   **Language:** TypeScript
--   **Module System:** ES Modules (`"type": "module"` in `package.json`)
--   **Compiler Target:** ES2020
--   **Code Style:** The code is well-structured and follows standard TypeScript conventions. It uses classes for services and the main server logic.
--   **Error Handling:** The code includes error handling for API requests and other operations.
--   **Caching:** Caching is implemented for both search results and webpage content to improve performance and reduce API calls.
--   **Testing:** There are no explicit test files in the provided structure.

package/QWEN.md

@@ -1,207 +0,0 @@
-# Google Search MCP Server - Project Context
-
-## Project Overview
-
-The Google Search MCP Server is a Model Context Protocol (MCP) server that provides Google search capabilities and webpage content analysis tools. This server enables AI models to perform Google searches and analyze webpage content programmatically through a standardized MCP interface.
-
-### Key Features
-- Google Custom Search integration
-- Advanced search features (filters, sorting, pagination, categorization)
-- Webpage content analysis in multiple formats (markdown, HTML, plain text)
-- Batch webpage analysis
-- Result categorization and classification
-- Content summarization
-- Optimized, human-readable responses
-- MCP-compliant interface
-
-### Technology Stack
-- **Language**: TypeScript
-- **Framework**: Node.js
-- **Core Dependencies**:
-  - `@modelcontextprotocol/sdk`: MCP server SDK
-  - `googleapis`: Google API integration
-  - `@mozilla/readability`: Content extraction
-  - `cheerio`: HTML parsing
-  - `jsdom`: DOM manipulation
-  - `turndown`: HTML to Markdown conversion
-  - `axios`: HTTP client
-  - `express`: Web framework (if needed)
-
-## Project Structure
-
-```
-D:\ai\pse-mcp\
-├── dist/                 # Compiled JavaScript files
-├── dist-package/         # Distribution package
-├── MCP Documents/        # MCP protocol documentation
-├── src/                  # Source TypeScript files
-│   ├── services/         # Service implementations
-│   │   ├── google-search.service.ts
-│   │   └── content-extractor.service.ts
-│   ├── google-search.ts  # Main server entry point
-│   ├── mcp.d.ts          # MCP type definitions
-│   └── types.ts          # Type definitions
-├── package.json          # Project dependencies and scripts
-├── tsconfig.json         # TypeScript configuration
-├── README.md             # Project documentation
-└── ...
-```
-
-## Building and Running
-
-### Prerequisites
-- Node.js (v16 or higher)
-- Google Cloud Platform account
-- Custom Search Engine ID
-- Google API Key
-
-### Setup Commands
-1. Install dependencies:
-   ```bash
-   npm install
-   ```
-
-2. Build the TypeScript code:
-   ```bash
-   npm run build
-   ```
-
-3. Run the server:
-   ```bash
-   npm run start
-   ```
-
-### Environment Variables
-Required environment variables:
-- `GOOGLE_API_KEY`: Your Google API key
-- `GOOGLE_SEARCH_ENGINE_ID`: Your Custom Search Engine ID
-
-### Development Scripts
-- `npm run build`: Compiles TypeScript to JavaScript
-- `npm run start`: Runs the compiled server
-- `npm run dev`: Watches for changes and recompiles (tsc -w)
-
-## Architecture and Components
-
-### Main Components
-1. **GoogleSearchService**: Handles Google API interactions for search functionality
-2. **ContentExtractor**: Manages webpage content analysis and extraction
-3. **Main Server (google-search.ts)**: MCP server implementation with tool handlers
-
-### Services
-
-#### GoogleSearchService
-- Integrates with Google Custom Search API
-- Provides caching mechanism (5-minute TTL, max 100 entries)
-- Implements advanced search filtering and pagination
-- Categorizes search results by content type
-- Handles error management and response formatting
-
-#### ContentExtractor
-- Extracts webpage content using Mozilla Readability
-- Converts content to markdown, HTML, or plain text formats
-- Implements content caching (30-minute TTL, max 50 entries)
-- Generates content summaries and statistics
-- Handles batch processing of multiple webpages
-
-### Available Tools
-
-#### 1. google_search
-Searches Google and returns relevant results with advanced filtering options:
-- Query string (required)
-- Number of results (default: 5, max: 10)
-- Site filtering
-- Language filtering (ISO 639-1 codes)
-- Date restrictions
-- Exact phrase matching
-- Result type (news, images, videos)
-- Pagination support
-- Sorting (relevance or date)
-
-#### 2. extract_webpage_content
-Extracts and analyzes content from a single webpage:
-- URL (required)
-- Output format (markdown, html, text)
-- Removes ads, navigation, and clutter
-- Returns title, description, content stats, and summary
-
-#### 3. extract_multiple_webpages
-Extracts content from multiple webpages in a single request:
-- Array of URLs (max 5 per request)
-- Output format (markdown, html, text)
-- Batch processing capability
-
-## Configuration
-
-The server configuration needs to be added to the MCP settings file (typically located at `%APPDATA%/Code/User/globalStorage/saoudrizwan.claude-dev/settings/cline_mcp_settings.json`):
-
-```json
-{
-  "mcpServers": {
-    "google-search": {
-      "autoApprove": [
-        "google_search",
-        "extract_webpage_content",
-        "extract_multiple_webpages"
-      ],
-      "disabled": false,
-      "timeout": 60,
-      "command": "node",
-      "args": [
-        "/path/to/google-search-mcp-server/dist/google-search.js"
-      ],
-      "env": {
-        "GOOGLE_API_KEY": "your-google-api-key",
-        "GOOGLE_SEARCH_ENGINE_ID": "your-custom-search-engine-id"
-      },
-      "transportType": "stdio"
-    }
-  }
-}
-```
-
-## Development Conventions
-
-### Coding Standards
-- TypeScript with strict mode enabled
-- Use of async/await for asynchronous operations
-- Proper error handling with descriptive messages
-- Input validation for all tool arguments
-- Caching strategies for performance optimization
-
-### Type Safety
-- Comprehensive type definitions in `types.ts`
-- Strict typing for all function parameters and return values
-- MCP request/response schema validation
-
-### Caching Strategy
-- Search results: 5-minute TTL with max 100 entries
-- Webpage content: 30-minute TTL with max 50 entries
-- Cache keys generated from request parameters
-- Automatic cleanup of oldest entries when limits exceeded
-
-## MCP Protocol Integration
-
-The server implements the Model Context Protocol with:
-- Standardized tool listing and calling
-- Input schema validation
-- Error response formatting
-- Stdio transport for communication with MCP clients
-
-## Testing and Verification
-
-To verify the server is working:
-1. Build with `npm run build`
-2. Start with `npm run start`
-3. Use MCP client to call the available tools
-4. Verify search results and content extraction work as expected
-
-## Deployment
-
-For distribution, the project includes a process to create a compiled distribution package:
-1. Build the TypeScript code
-2. Create a distribution package with only necessary files
-3. Include production dependencies only
-4. Simplified package.json for end users
-
-The distribution approach allows for shipping compiled JavaScript without exposing source code while maintaining functionality.