qwksearch-mcp-server 1.0.28 → 1.0.30

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "qwksearch-mcp-server",
-  "version": "1.0.28",
+  "version": "1.0.30",
   "description": "MCP server exposing web search and content extraction tools via stdio transport",
   "repository": {
     "type": "git",
@@ -18,12 +18,18 @@
   },
   "scripts": {
     "start": "bun run bin/qwksearch-mcp.ts",
-    "dev": "bun --watch run bin/qwksearch-mcp.ts"
+    "dev": "bun --watch run bin/qwksearch-mcp.ts",
+    "test": "vitest run",
+    "test:coverage": "vitest run --coverage"
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.30.0",
     "search-web-api": "^1.0.76",
-    "extract-webpage": "^1.2.168",
+    "extract-webpage": "^1.2.170",
     "zod": "^4.3.6"
+  },
+  "devDependencies": {
+    "@vitest/coverage-v8": "^4.0.18",
+    "vitest": "^4.0.18"
   }
 }

package/test/extract-page.test.ts

@@ -0,0 +1,120 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { createFakeServer } from './helpers/fake-server';
+
+const extractContent = vi.fn();
+vi.mock('extract-webpage/url-to-content/url-to-content', () => ({ extractContent }));
+
+const { registerExtractPageTool } = await import('../src/tools/extract-page.js');
+
+function register() {
+  const fake = createFakeServer();
+  registerExtractPageTool(fake.server);
+  return fake.get('extract_page');
+}
+
+const DEFAULT_ARGS = { url: 'https://example.com/article', images: true, links: true, formatting: true, timeout: 10 };
+
+describe('extract_page tool registration', () => {
+  it('registers under the expected name', () => {
+    expect(register().name).toBe('extract_page');
+  });
+
+  it('declares the extraction options as inputs', () => {
+    const schema = register().config.inputSchema;
+
+    expect(Object.keys(schema).sort()).toEqual(['formatting', 'images', 'links', 'timeout', 'url']);
+  });
+});
+
+describe('extract_page handler', () => {
+  beforeEach(() => {
+    extractContent.mockReset();
+  });
+
+  it('forwards the options and always requests absolute URLs', async () => {
+    extractContent.mockResolvedValue({ title: 'Article' });
+
+    await register().handler({ ...DEFAULT_ARGS, images: false, timeout: 30 });
+
+    expect(extractContent).toHaveBeenCalledWith('https://example.com/article', {
+      images: false,
+      links: true,
+      formatting: true,
+      absoluteURLs: true,
+      timeout: 30,
+    });
+  });
+
+  it('renders the full set of metadata fields', async () => {
+    extractContent.mockResolvedValue({
+      url: 'https://example.com/canonical',
+      title: 'Article',
+      author: 'A. Writer',
+      date: '2024-01-01',
+      source: 'example.com',
+      word_count: 812,
+      cite: 'Writer, A. (2024)',
+      html: '<p>Body</p>',
+    });
+
+    const text = (await register().handler(DEFAULT_ARGS)).content[0].text;
+
+    expect(text).toContain('Content extracted from: https://example.com/canonical');
+    expect(text).toContain('Title: Article');
+    expect(text).toContain('Author: A. Writer');
+    expect(text).toContain('Date: 2024-01-01');
+    expect(text).toContain('Source: example.com');
+    expect(text).toContain('Word Count: 812');
+    expect(text).toContain('Citation: Writer, A. (2024)');
+    expect(text).toContain('<p>Body</p>');
+  });
+
+  it('falls back to the requested URL when the result has none', async () => {
+    extractContent.mockResolvedValue({ title: 'Article' });
+
+    const text = (await register().handler(DEFAULT_ARGS)).content[0].text;
+
+    expect(text).toContain('Content extracted from: https://example.com/article');
+  });
+
+  it('omits metadata lines for fields the extractor did not return', async () => {
+    extractContent.mockResolvedValue({ title: 'Article' });
+
+    const text = (await register().handler(DEFAULT_ARGS)).content[0].text;
+
+    expect(text).not.toContain('Author:');
+    expect(text).not.toContain('Citation:');
+    expect(text).not.toContain('Word Count:');
+  });
+
+  it('flags an extractor-reported error', async () => {
+    extractContent.mockResolvedValue({ error: 'paywalled' });
+
+    const result = await register().handler(DEFAULT_ARGS);
+
+    expect(result.isError).toBe(true);
+    expect(result.content[0].text).toBe(
+      'Could not extract content from "https://example.com/article".'
+    );
+  });
+
+  it('flags an empty extractor result', async () => {
+    extractContent.mockResolvedValue(null);
+
+    const result = await register().handler(DEFAULT_ARGS);
+
+    expect(result.isError).toBe(true);
+    expect(result.content[0].text).toContain('Could not extract content');
+  });
+
+  it('reports a thrown error with its message', async () => {
+    extractContent.mockRejectedValue(new Error('DNS failure'));
+
+    const result = await register().handler(DEFAULT_ARGS);
+
+    expect(result.isError).toBe(true);
+    expect(result.content[0].text).toBe(
+      'Extraction failed for "https://example.com/article": DNS failure'
+    );
+  });
+});

package/test/helpers/fake-server.ts

@@ -0,0 +1,37 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+
+export type RegisteredTool = {
+  name: string;
+  config: {
+    description: string;
+    inputSchema: Record<string, unknown>;
+  };
+  handler: (args: Record<string, unknown>) => Promise<{
+    content: { type: 'text'; text: string }[];
+    isError?: boolean;
+  }>;
+};
+
+/**
+ * Captures `registerTool` calls so a tool's handler can be invoked directly,
+ * without standing up an MCP transport.
+ */
+export function createFakeServer() {
+  const tools = new Map<string, RegisteredTool>();
+
+  const server = {
+    registerTool(name: string, config: RegisteredTool['config'], handler: RegisteredTool['handler']) {
+      tools.set(name, { name, config, handler });
+    },
+  };
+
+  return {
+    server: server as unknown as McpServer,
+    tools,
+    get(name: string): RegisteredTool {
+      const tool = tools.get(name);
+      if (!tool) throw new Error(`Tool "${name}" was never registered`);
+      return tool;
+    },
+  };
+}

package/test/render-page.test.ts

@@ -0,0 +1,145 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { registerRenderPageTool } from '../src/tools/render-page.js';
+import { createFakeServer } from './helpers/fake-server';
+
+function register() {
+  const fake = createFakeServer();
+  registerRenderPageTool(fake.server);
+  return fake.get('render_page_with_javascript');
+}
+
+const DEFAULT_ARGS = {
+  url: 'https://example.com/app',
+  wait: 0,
+  timeout: 30000,
+  waitUntil: 'networkidle2',
+};
+
+function mockFetch(body: unknown, init: { ok?: boolean; text?: string } = {}) {
+  const fetchMock = vi.fn(async () => ({
+    ok: init.ok ?? true,
+    json: async () => body,
+    text: async () => init.text ?? '',
+  }));
+  vi.stubGlobal('fetch', fetchMock);
+  return fetchMock;
+}
+
+describe('render_page_with_javascript tool registration', () => {
+  it('registers under the expected name', () => {
+    expect(register().name).toBe('render_page_with_javascript');
+  });
+
+  it('declares the render options as inputs', () => {
+    const schema = register().config.inputSchema;
+
+    expect(Object.keys(schema).sort()).toEqual(['timeout', 'url', 'wait', 'waitUntil']);
+  });
+});
+
+describe('render_page_with_javascript handler', () => {
+  beforeEach(() => {
+    vi.unstubAllEnvs();
+    vi.unstubAllGlobals();
+  });
+
+  it('refuses to run without SCRAPER_API_KEY', async () => {
+    vi.stubEnv('SCRAPER_API_KEY', '');
+
+    const result = await register().handler(DEFAULT_ARGS);
+
+    expect(result.isError).toBe(true);
+    expect(result.content[0].text).toContain('SCRAPER_API_KEY environment variable not set');
+  });
+
+  it('posts to the default scraper with a bearer token', async () => {
+    vi.stubEnv('SCRAPER_API_KEY', 'secret-key');
+    const fetchMock = mockFetch({ html: '<html></html>' });
+
+    await register().handler(DEFAULT_ARGS);
+
+    const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit];
+    expect(url).toBe('https://scraper.qwksearch.workers.dev/api/render');
+    expect(init.method).toBe('POST');
+    expect((init.headers as Record<string, string>).Authorization).toBe('Bearer secret-key');
+  });
+
+  it('honours a SCRAPER_URL override', async () => {
+    vi.stubEnv('SCRAPER_API_KEY', 'secret-key');
+    vi.stubEnv('SCRAPER_URL', 'https://scraper.internal.example');
+    const fetchMock = mockFetch({ html: '' });
+
+    await register().handler(DEFAULT_ARGS);
+
+    expect(fetchMock.mock.calls[0][0]).toBe('https://scraper.internal.example/api/render');
+  });
+
+  it('forwards the render options in the request body', async () => {
+    vi.stubEnv('SCRAPER_API_KEY', 'secret-key');
+    const fetchMock = mockFetch({ html: '' });
+
+    await register().handler({ ...DEFAULT_ARGS, wait: 500, timeout: 45000, waitUntil: 'load' });
+
+    const body = JSON.parse((fetchMock.mock.calls[0][1] as unknown as RequestInit).body as string);
+    expect(body).toMatchObject({
+      url: 'https://example.com/app',
+      wait: 500,
+      timeout: 45000,
+      waitUntil: 'load',
+      blockImages: true,
+      bypassCaptcha: true,
+      format: 'json',
+    });
+  });
+
+  it('renders the page title, load time and html', async () => {
+    vi.stubEnv('SCRAPER_API_KEY', 'secret-key');
+    mockFetch({
+      url: 'https://example.com/app/final',
+      title: 'My App',
+      loadTime: 1234,
+      html: '<h1>Hi</h1>',
+    });
+
+    const text = (await register().handler(DEFAULT_ARGS)).content[0].text;
+
+    expect(text).toContain('Page rendered: https://example.com/app/final');
+    expect(text).toContain('Title: My App');
+    expect(text).toContain('Load Time: 1234ms');
+    expect(text).toContain('<h1>Hi</h1>');
+  });
+
+  it('falls back to the requested URL and tolerates missing html', async () => {
+    vi.stubEnv('SCRAPER_API_KEY', 'secret-key');
+    mockFetch({});
+
+    const text = (await register().handler(DEFAULT_ARGS)).content[0].text;
+
+    expect(text).toContain('Page rendered: https://example.com/app');
+    expect(text).not.toContain('Title:');
+  });
+
+  it('surfaces the scraper error body on a non-ok response', async () => {
+    vi.stubEnv('SCRAPER_API_KEY', 'secret-key');
+    mockFetch(null, { ok: false, text: 'quota exceeded' });
+
+    const result = await register().handler(DEFAULT_ARGS);
+
+    expect(result.isError).toBe(true);
+    expect(result.content[0].text).toBe('Render failed: quota exceeded');
+  });
+
+  it('reports a thrown error with the requested URL', async () => {
+    vi.stubEnv('SCRAPER_API_KEY', 'secret-key');
+    vi.stubGlobal('fetch', vi.fn(async () => {
+      throw new Error('socket hang up');
+    }));
+
+    const result = await register().handler(DEFAULT_ARGS);
+
+    expect(result.isError).toBe(true);
+    expect(result.content[0].text).toBe(
+      'Render failed for "https://example.com/app": socket hang up'
+    );
+  });
+});

package/test/search-instance.test.ts

@@ -0,0 +1,26 @@
+import { describe, expect, it, vi } from 'vitest';
+
+const constructed: unknown[] = [];
+class FakeSearch {
+  constructor() {
+    constructed.push(this);
+  }
+}
+
+vi.mock('search-web-api', () => ({ Search: FakeSearch }));
+
+const { getSearchInstance } = await import('../src/lib/search-instance.js');
+
+describe('getSearchInstance', () => {
+  it('returns a Search instance', () => {
+    expect(getSearchInstance()).toBeInstanceOf(FakeSearch);
+  });
+
+  it('constructs the instance lazily and only once', () => {
+    const first = getSearchInstance();
+    const second = getSearchInstance();
+
+    expect(second).toBe(first);
+    expect(constructed).toHaveLength(1);
+  });
+});

package/test/web-search.test.ts

@@ -0,0 +1,114 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { createFakeServer } from './helpers/fake-server';
+
+const search = vi.fn();
+vi.mock('../src/lib/search-instance.js', () => ({
+  getSearchInstance: () => ({ search }),
+}));
+
+const { registerWebSearchTool } = await import('../src/tools/web-search.js');
+
+function register() {
+  const fake = createFakeServer();
+  registerWebSearchTool(fake.server);
+  return fake.get('web_search');
+}
+
+describe('web_search tool registration', () => {
+  it('registers under the expected name with a description', () => {
+    const tool = register();
+
+    expect(tool.name).toBe('web_search');
+    expect(tool.config.description).toContain('Search the web');
+  });
+
+  it('declares query, category and page inputs', () => {
+    const tool = register();
+
+    expect(Object.keys(tool.config.inputSchema).sort()).toEqual(['category', 'page', 'query']);
+  });
+});
+
+describe('web_search handler', () => {
+  beforeEach(() => {
+    search.mockReset();
+  });
+
+  it('passes the query, page and category through to the search instance', async () => {
+    search.mockResolvedValue([{ title: 'A result', url: 'https://example.com' }]);
+
+    await register().handler({ query: 'espresso', category: 'news', page: 2 });
+
+    expect(search).toHaveBeenCalledWith('espresso', 2, undefined, ['news']);
+  });
+
+  it('formats results as a numbered list with URLs', async () => {
+    search.mockResolvedValue([
+      { title: 'First', url: 'https://example.com/1', content: 'Snippet one' },
+      { title: 'Second', url: 'https://example.com/2' },
+    ]);
+
+    const result = await register().handler({ query: 'espresso', category: 'general', page: 1 });
+    const text = result.content[0].text;
+
+    expect(text).toContain('Search results for "espresso" (general):');
+    expect(text).toContain('1. First');
+    expect(text).toContain('URL: https://example.com/1');
+    expect(text).toContain('Snippet one');
+    expect(text).toContain('2. Second');
+    expect(text).toContain('Found 2 results.');
+    expect(result.isError).toBeUndefined();
+  });
+
+  it('falls back to the `link` field when `url` is absent', async () => {
+    search.mockResolvedValue([{ title: 'Legacy', link: 'https://example.com/legacy' }]);
+
+    const result = await register().handler({ query: 'x', category: 'general', page: 1 });
+
+    expect(result.content[0].text).toContain('URL: https://example.com/legacy');
+  });
+
+  it('lists the contributing engines when present', async () => {
+    search.mockResolvedValue([
+      { title: 'A', url: 'https://example.com', engines: ['google', 'brave'] },
+    ]);
+
+    const result = await register().handler({ query: 'x', category: 'general', page: 1 });
+
+    expect(result.content[0].text).toContain('Sources: google, brave');
+  });
+
+  it('omits the sources line when the engine list is empty', async () => {
+    search.mockResolvedValue([{ title: 'A', url: 'https://example.com', engines: [] }]);
+
+    const result = await register().handler({ query: 'x', category: 'general', page: 1 });
+
+    expect(result.content[0].text).not.toContain('Sources:');
+  });
+
+  it('reports an empty result set without flagging an error', async () => {
+    search.mockResolvedValue([]);
+
+    const result = await register().handler({ query: 'nothing here', category: 'general', page: 1 });
+
+    expect(result.content[0].text).toBe('No search results found for "nothing here".');
+    expect(result.isError).toBeUndefined();
+  });
+
+  it('treats a null result the same as an empty one', async () => {
+    search.mockResolvedValue(null);
+
+    const result = await register().handler({ query: 'nothing', category: 'general', page: 1 });
+
+    expect(result.content[0].text).toContain('No search results found');
+  });
+
+  it('returns an error payload when the search throws', async () => {
+    search.mockRejectedValue(new Error('upstream timeout'));
+
+    const result = await register().handler({ query: 'x', category: 'general', page: 1 });
+
+    expect(result.isError).toBe(true);
+    expect(result.content[0].text).toBe('Search failed: upstream timeout');
+  });
+});

package/vitest.config.ts

@@ -0,0 +1,25 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    globals: true,
+    environment: 'node',
+    include: ['test/**/*.test.ts'],
+    restoreMocks: true,
+    unstubEnvs: true,
+    unstubGlobals: true,
+    coverage: {
+      provider: 'v8',
+      reporter: ['text', 'json', 'html', 'lcov'],
+      include: ['src/**/*.ts'],
+      exclude: [
+        'node_modules/**',
+        'dist/**',
+        // src/index.ts opens a real stdio transport on import, so it is
+        // covered by the bin smoke path rather than by unit tests.
+        'src/index.ts',
+        '**/*.test.ts',
+      ],
+    },
+  },
+});