qwksearch-mcp-server 1.0.0

package/bin/qwksearch-mcp.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env bun
+import "../src/index.js";

package/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "qwksearch-mcp-server",
+  "version": "1.0.0",
+  "description": "MCP server exposing web search and content extraction tools via stdio transport",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/OpenSourceAGI/qwksearch-research-agent.git",
+    "directory": "packages/qwksearch-mcp-server"
+  },
+  "type": "module",
+  "bin": {
+    "qwksearch-mcp": "./bin/qwksearch-mcp.ts"
+  },
+  "exports": {
+    ".": {
+      "import": "./src/index.ts"
+    }
+  },
+  "scripts": {
+    "start": "bun run bin/qwksearch-mcp.ts",
+    "dev": "bun --watch run bin/qwksearch-mcp.ts"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.30.0",
+    "search-web-api": "^1.0.76",
+    "extract-webpage": "^1.2.140",
+    "zod": "^4.3.6"
+  }
+}

package/src/index.ts

@@ -0,0 +1,19 @@
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { registerWebSearchTool } from "./tools/web-search.js";
+import { registerExtractPageTool } from "./tools/extract-page.js";
+import { registerRenderPageTool } from "./tools/render-page.js";
+
+const server = new McpServer({
+  name: "qwksearch",
+  version: "1.0.0",
+}, {
+  capabilities: { tools: {} },
+});
+
+registerWebSearchTool(server);
+registerExtractPageTool(server);
+registerRenderPageTool(server);
+
+const transport = new StdioServerTransport();
+await server.connect(transport);

package/src/lib/search-instance.ts

@@ -0,0 +1,10 @@
+import { Search } from "search-web-api";
+
+let instance: Search | null = null;
+
+export function getSearchInstance(): Search {
+  if (!instance) {
+    instance = new Search();
+  }
+  return instance;
+}

package/src/tools/extract-page.ts

@@ -0,0 +1,50 @@
+import { z } from "zod";
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { extractContent } from "extract-webpage/url-to-content/url-to-content";
+
+export function registerExtractPageTool(server: McpServer) {
+  server.registerTool("extract_page", {
+    description:
+      "Extract content from a web page, PDF, or YouTube video. Uses Mozilla Readability and Postlight Mercury algorithms with 100+ custom adapters. Returns structured content with title, author, date, and citation.",
+    inputSchema: {
+      url: z.string().describe("URL of the page to extract"),
+      images: z.boolean().optional().default(true).describe("Include images in output"),
+      links: z.boolean().optional().default(true).describe("Include links in output"),
+      formatting: z.boolean().optional().default(true).describe("Preserve formatting"),
+      timeout: z.number().optional().default(10).describe("HTTP request timeout in seconds"),
+    },
+  }, async ({ url, images, links, formatting, timeout }) => {
+    try {
+      const result = await extractContent(url, {
+        images,
+        links,
+        formatting,
+        absoluteURLs: true,
+        timeout,
+      });
+
+      if (!result || result.error) {
+        return {
+          content: [{ type: "text" as const, text: `Could not extract content from "${url}".` }],
+          isError: true,
+        };
+      }
+
+      let text = `Content extracted from: ${result.url || url}\n\n`;
+      if (result.title) text += `Title: ${result.title}\n`;
+      if (result.author) text += `Author: ${result.author}\n`;
+      if (result.date) text += `Date: ${result.date}\n`;
+      if (result.source) text += `Source: ${result.source}\n`;
+      if (result.word_count) text += `Word Count: ${result.word_count}\n`;
+      if (result.cite) text += `Citation: ${result.cite}\n`;
+      if (result.html) text += `\nContent:\n${result.html}\n`;
+
+      return { content: [{ type: "text" as const, text }] };
+    } catch (error: any) {
+      return {
+        content: [{ type: "text" as const, text: `Extraction failed for "${url}": ${error.message}` }],
+        isError: true,
+      };
+    }
+  });
+}

package/src/tools/render-page.ts

@@ -0,0 +1,71 @@
+import { z } from "zod";
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+
+export function registerRenderPageTool(server: McpServer) {
+  server.registerTool("render_page_with_javascript", {
+    description:
+      "Render a web page with full JavaScript execution using Cloudflare Browser Rendering. Use for JavaScript-heavy sites (SPAs, React apps) or pages behind bot protection. Slower but more complete than extract_page.",
+    inputSchema: {
+      url: z.string().describe("URL of the page to render"),
+      wait: z.number().optional().default(0).describe("Additional wait time in ms after page load"),
+      timeout: z.number().optional().default(30000).describe("Timeout in ms"),
+      waitUntil: z
+        .enum(["domcontentloaded", "load", "networkidle0", "networkidle2"])
+        .optional()
+        .default("networkidle2")
+        .describe("When to consider navigation complete"),
+    },
+  }, async ({ url, wait, timeout, waitUntil }) => {
+    const scraperUrl = process.env.SCRAPER_URL || "https://scraper.qwksearch.workers.dev";
+    const scraperApiKey = process.env.SCRAPER_API_KEY;
+
+    if (!scraperApiKey) {
+      return {
+        content: [{ type: "text" as const, text: "render_page_with_javascript is unavailable: SCRAPER_API_KEY environment variable not set." }],
+        isError: true,
+      };
+    }
+
+    try {
+      const requestUrl = new URL("/api/render", scraperUrl);
+      const response = await fetch(requestUrl.toString(), {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${scraperApiKey}`,
+        },
+        body: JSON.stringify({
+          url,
+          blockImages: true,
+          wait,
+          timeout,
+          waitUntil,
+          bypassCaptcha: true,
+          sessionId: "default",
+          format: "json",
+        }),
+      });
+
+      if (!response.ok) {
+        const errorText = await response.text();
+        return {
+          content: [{ type: "text" as const, text: `Render failed: ${errorText}` }],
+          isError: true,
+        };
+      }
+
+      const data = await response.json() as any;
+      let text = `Page rendered: ${data.url || url}\n`;
+      if (data.title) text += `Title: ${data.title}\n`;
+      if (data.loadTime) text += `Load Time: ${data.loadTime}ms\n`;
+      text += `\n${data.html || ""}`;
+
+      return { content: [{ type: "text" as const, text }] };
+    } catch (error: any) {
+      return {
+        content: [{ type: "text" as const, text: `Render failed for "${url}": ${error.message}` }],
+        isError: true,
+      };
+    }
+  });
+}

package/src/tools/web-search.ts

@@ -0,0 +1,51 @@
+import { z } from "zod";
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { getSearchInstance } from "../lib/search-instance.js";
+
+export function registerWebSearchTool(server: McpServer) {
+  server.registerTool("web_search", {
+    description:
+      "Search the web for information on any topic. Returns relevant search results with titles, descriptions, and URLs from 75+ search engines across 10 categories.",
+    inputSchema: {
+      query: z.string().describe("Search query string"),
+      category: z
+        .enum(["general", "news", "videos", "images", "science", "files", "it", "academic", "torrents", "social", "maps", "shopping"])
+        .optional()
+        .default("general")
+        .describe("Search category to focus results"),
+      page: z
+        .number()
+        .optional()
+        .default(1)
+        .describe("Result page number"),
+    },
+  }, async ({ query, category, page }) => {
+    try {
+      const search = getSearchInstance();
+      const results = await search.search(query, page, undefined, [category]);
+
+      if (!results || results.length === 0) {
+        return {
+          content: [{ type: "text" as const, text: `No search results found for "${query}".` }],
+        };
+      }
+
+      let text = `Search results for "${query}" (${category}):\n\n`;
+      for (const [i, r] of results.entries()) {
+        text += `${i + 1}. ${r.title}\n`;
+        text += `   URL: ${r.url || r.link}\n`;
+        if (r.content) text += `   ${r.content}\n`;
+        if (r.engines?.length) text += `   Sources: ${r.engines.join(", ")}\n`;
+        text += "\n";
+      }
+      text += `Found ${results.length} results.`;
+
+      return { content: [{ type: "text" as const, text }] };
+    } catch (error: any) {
+      return {
+        content: [{ type: "text" as const, text: `Search failed: ${error.message}` }],
+        isError: true,
+      };
+    }
+  });
+}

package/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "outDir": "./dist",
+    "rootDir": "./src"
+  },
+  "include": ["src/**/*", "bin/**/*"]
+}