peeky-search 1.0.10

package/dist/cli.js

@@ -0,0 +1,357 @@
+#!/usr/bin/env node
+import {
+  createConfigFiles,
+  getConfigDir,
+  getMcpConfigJson,
+  readConfig,
+  startContainer,
+  stopContainer,
+  waitForReady
+} from "./chunk-F3PNR32Z.js";
+
+// src/setup/checks.ts
+import { exec } from "child_process";
+import { promisify } from "util";
+import * as net from "net";
+var execAsync = promisify(exec);
+async function checkDockerInstalled() {
+  try {
+    await execAsync("docker --version");
+    return { success: true, message: "Docker is installed" };
+  } catch {
+    return {
+      success: false,
+      message: "Docker not found. Install from https://docker.com"
+    };
+  }
+}
+async function checkDockerRunning() {
+  try {
+    await execAsync("docker info");
+    return { success: true, message: "Docker is running" };
+  } catch {
+    return {
+      success: false,
+      message: "Docker is not running. Please start Docker Desktop."
+    };
+  }
+}
+async function checkPortAvailable(port) {
+  return new Promise((resolve) => {
+    const server = net.createServer();
+    server.once("error", (err) => {
+      if (err.code === "EADDRINUSE") {
+        resolve({
+          success: false,
+          message: `Port ${port} is in use. Use --port to specify another.`
+        });
+      } else {
+        resolve({
+          success: false,
+          message: `Port check failed: ${err.message}`
+        });
+      }
+    });
+    server.once("listening", () => {
+      server.close(() => {
+        resolve({ success: true, message: `Port ${port} is available` });
+      });
+    });
+    server.listen(port, "127.0.0.1");
+  });
+}
+async function runAllChecks(port) {
+  const results = [];
+  const dockerInstalled = await checkDockerInstalled();
+  results.push(dockerInstalled);
+  if (!dockerInstalled.success) {
+    return { allPassed: false, results };
+  }
+  const dockerRunning = await checkDockerRunning();
+  results.push(dockerRunning);
+  if (!dockerRunning.success) {
+    return { allPassed: false, results };
+  }
+  const portAvailable = await checkPortAvailable(port);
+  results.push(portAvailable);
+  return {
+    allPassed: results.every((r) => r.success),
+    results
+  };
+}
+
+// src/setup/index.ts
+var DEFAULT_PORT = 8888;
+function printStep(success, message) {
+  const symbol = success ? "\u2713" : "\u2717";
+  console.log(`  ${symbol} ${message}`);
+}
+function printSection(title) {
+  console.log(`
+  ${title}`);
+}
+async function runSetup(options = {}) {
+  const port = options.port ?? DEFAULT_PORT;
+  console.log("\npeeky-search setup");
+  console.log("=".repeat(60));
+  printSection("Checking prerequisites...");
+  const { allPassed, results } = await runAllChecks(port);
+  for (const result of results) {
+    printStep(result.success, result.message);
+  }
+  if (!allPassed) {
+    console.log("\n  Setup cannot continue. Please fix the issues above.\n");
+    return false;
+  }
+  if (options.checkOnly) {
+    console.log("\n  All prerequisites passed!\n");
+    return true;
+  }
+  printSection("Setting up SearXNG...");
+  try {
+    createConfigFiles(port);
+    printStep(true, `Created ${getConfigDir()}/docker-compose.yml`);
+    printStep(true, `Created ${getConfigDir()}/settings.yml (secret key generated)`);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    printStep(false, `Failed to create config files: ${message}`);
+    return false;
+  }
+  const startResult = await startContainer();
+  if (!startResult.success) {
+    printStep(false, startResult.message);
+    return false;
+  }
+  printStep(true, "Started SearXNG container");
+  console.log("  Waiting for SearXNG to be ready...");
+  const ready = await waitForReady(port, 6e4);
+  if (!ready) {
+    printStep(false, "SearXNG did not respond within 60 seconds");
+    console.log("  Try running 'peeky-search status' to check the container.\n");
+    return false;
+  }
+  printStep(true, `SearXNG is responding at http://localhost:${port}`);
+  console.log("\n  \u2705 Setup complete!");
+  console.log("\n  Add this to your MCP client config:");
+  console.log("  " + "-".repeat(56));
+  const configJson = getMcpConfigJson(port);
+  for (const line of configJson.split("\n")) {
+    console.log("  " + line);
+  }
+  console.log("  " + "-".repeat(56));
+  console.log("\n  Commands:");
+  console.log("    peeky-search start      Start the SearXNG container");
+  console.log("    peeky-search stop       Stop the SearXNG container");
+  console.log("    peeky-search status     Check if SearXNG is running");
+  console.log("    peeky-search uninstall  Remove container and config");
+  console.log("\n  Requirements:");
+  console.log("    Docker must be running for SearXNG to work.");
+  console.log("    The container runs in the background and restarts automatically.");
+  console.log("");
+  return true;
+}
+async function printStatus() {
+  const config = readConfig();
+  console.log("\npeeky-search status");
+  console.log("=".repeat(60));
+  if (!config) {
+    console.log("  Not installed. Run 'peeky-search setup' first.\n");
+    return;
+  }
+  console.log(`  Config directory: ${getConfigDir()}`);
+  console.log(`  Port: ${config.port}`);
+  console.log(`  Installed: ${config.installedAt}`);
+  const { getStatus } = await import("./docker-IWGZDSIP.js");
+  const status = await getStatus();
+  console.log("");
+  printStep(status.containerRunning, `Container ${status.containerRunning ? "running" : "not running"}`);
+  printStep(
+    status.searxngResponding,
+    `SearXNG ${status.searxngResponding ? "responding" : "not responding"} at http://localhost:${config.port}`
+  );
+  if (!status.containerRunning) {
+    console.log("\n  Run 'peeky-search start' to start the container.\n");
+  } else {
+    console.log("");
+  }
+}
+
+// src/setup/uninstall.ts
+import * as fs from "fs";
+async function uninstall() {
+  const steps = [];
+  const configDir = getConfigDir();
+  const stopResult = await stopContainer();
+  steps.push({
+    step: "Stop container",
+    success: stopResult.success,
+    message: stopResult.message
+  });
+  if (fs.existsSync(configDir)) {
+    try {
+      fs.rmSync(configDir, { recursive: true, force: true });
+      steps.push({
+        step: "Remove config",
+        success: true,
+        message: `Removed ${configDir}`
+      });
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      steps.push({
+        step: "Remove config",
+        success: false,
+        message: `Failed to remove config: ${message}`
+      });
+    }
+  } else {
+    steps.push({
+      step: "Remove config",
+      success: true,
+      message: "Config directory not found (already removed)"
+    });
+  }
+  return {
+    success: steps.every((s) => s.success),
+    steps
+  };
+}
+
+// src/cli.ts
+var HELP_TEXT = `
+peeky-search - Web search tool for MCP (Model Context Protocol)
+
+Searches the web via a local SearXNG instance, scrapes pages, and extracts
+relevant excerpts using IR (information retrieval) techniques. Exposes a
+"peeky_web_search" tool to any MCP-compatible client.
+
+REQUIREMENTS:
+  Docker            Required to run SearXNG (the search backend)
+
+QUICK START:
+  npx peeky-search setup       One-time setup (starts SearXNG in Docker)
+  Then add the MCP config to your client and restart it.
+
+COMMANDS:
+  setup [options]    Install and start SearXNG
+    --port <port>      Use custom port (default: 8888)
+    --check            Check prerequisites only, don't install
+
+  start              Start the SearXNG container
+  stop               Stop the SearXNG container
+  status             Check if SearXNG is running
+  uninstall          Stop container and remove all config
+
+  mcp                Start the MCP server (called by MCP clients)
+  help, --help       Show this help message
+
+EXAMPLES:
+  npx peeky-search setup                    # First-time setup
+  npx peeky-search setup --port 9999        # Use a different port
+  npx peeky-search status                   # Check if running
+  npx peeky-search stop                     # Stop SearXNG
+  npx peeky-search start                    # Start SearXNG again
+  npx peeky-search uninstall                # Remove everything
+`;
+async function main() {
+  const args = process.argv.slice(2).filter((a) => a !== "--");
+  const command = args[0];
+  function parseSetupOptions() {
+    const options = {};
+    for (let i = 1; i < args.length; i++) {
+      const arg = args[i];
+      const nextArg = args[i + 1];
+      if (arg === "--port" && nextArg !== void 0) {
+        options.port = parseInt(nextArg, 10);
+        i++;
+      } else if (arg === "--check") {
+        options.checkOnly = true;
+      }
+    }
+    return options;
+  }
+  switch (command) {
+    case "setup": {
+      const options = parseSetupOptions();
+      const success = await runSetup(options);
+      process.exit(success ? 0 : 1);
+      break;
+    }
+    case "start": {
+      console.log("\nStarting SearXNG container...");
+      const result = await startContainer();
+      if (result.success) {
+        console.log("  \u2713 " + result.message);
+        console.log("\nRun 'peeky-search status' to verify.\n");
+      } else {
+        console.log("  \u2717 " + result.message + "\n");
+        process.exit(1);
+      }
+      break;
+    }
+    case "stop": {
+      console.log("\nStopping SearXNG container...");
+      const result = await stopContainer();
+      if (result.success) {
+        console.log("  \u2713 " + result.message + "\n");
+      } else {
+        console.log("  \u2717 " + result.message + "\n");
+        process.exit(1);
+      }
+      break;
+    }
+    case "status": {
+      await printStatus();
+      break;
+    }
+    case "uninstall": {
+      console.log("\npeeky-search uninstall");
+      console.log("=".repeat(60));
+      const result = await uninstall();
+      console.log("");
+      for (const step of result.steps) {
+        const symbol = step.success ? "\u2713" : "\u2717";
+        console.log(`  ${symbol} ${step.message}`);
+      }
+      if (result.success) {
+        console.log("\n  \u2705 Uninstall complete!");
+        console.log(
+          "\n  Note: Remember to remove peeky-search from your MCP client config.\n"
+        );
+      } else {
+        console.log("\n  Some steps failed. Check the errors above.\n");
+        process.exit(1);
+      }
+      break;
+    }
+    case "mcp": {
+      await import("./mcp/server.js");
+      break;
+    }
+    case "--help":
+    case "-h":
+    case "help": {
+      console.log(HELP_TEXT);
+      break;
+    }
+    case void 0: {
+      console.log(HELP_TEXT);
+      break;
+    }
+    default: {
+      if (command?.startsWith("--") || command?.startsWith("-")) {
+        await runExtraction(args);
+      } else {
+        console.log(`Unknown command: ${command}`);
+        console.log("Run 'peeky-search --help' for usage.\n");
+        process.exit(1);
+      }
+    }
+  }
+}
+async function runExtraction(_args) {
+  await import("./index.js");
+}
+main().catch((err) => {
+  console.error(`Unexpected error: ${err}`);
+  process.exit(1);
+});

package/dist/docker-IWGZDSIP.js

@@ -0,0 +1,20 @@
+import {
+  createConfigFiles,
+  getConfigDir,
+  getStatus,
+  readConfig,
+  startContainer,
+  stopContainer,
+  streamLogs,
+  waitForReady
+} from "./chunk-F3PNR32Z.js";
+export {
+  createConfigFiles,
+  getConfigDir,
+  getStatus,
+  readConfig,
+  startContainer,
+  stopContainer,
+  streamLogs,
+  waitForReady
+};

package/dist/index.js

@@ -0,0 +1,179 @@
+import {
+  extractExcerpts,
+  formatExcerpts,
+  logger_default,
+  search,
+  tokenize
+} from "./chunk-S3WZDJCP.js";
+
+// src/index.ts
+import * as fs from "fs";
+import * as path from "path";
+import { fileURLToPath } from "url";
+var __filename = fileURLToPath(import.meta.url);
+var __dirname = path.dirname(__filename);
+var logger = logger_default.getInstance();
+async function main() {
+  const args = process.argv.slice(2).filter((a) => a !== "--");
+  let query = "";
+  let filePath = "";
+  let url = "";
+  let searchMode = false;
+  let debug = false;
+  let timing = false;
+  let maxResults = 5;
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    const nextArg = args[i + 1];
+    if ((arg === "--query" || arg === "-q") && nextArg !== void 0) {
+      query = nextArg;
+      i++;
+    } else if ((arg === "--file" || arg === "-f") && nextArg !== void 0) {
+      filePath = nextArg;
+      i++;
+    } else if ((arg === "--url" || arg === "-u") && nextArg !== void 0) {
+      url = nextArg;
+      i++;
+    } else if (arg === "--search" || arg === "-s") {
+      searchMode = true;
+    } else if (arg === "--max" && nextArg !== void 0) {
+      maxResults = parseInt(nextArg, 10);
+      i++;
+    } else if (arg === "--debug") {
+      debug = true;
+    } else if (arg === "--timing" || arg === "-t") {
+      timing = true;
+    } else if (arg === "--help" || arg === "-h") {
+      printUsage();
+      return;
+    }
+  }
+  if (timing) {
+    logger.setTimingEnabled(true);
+  }
+  if (searchMode) {
+    if (query === "") {
+      logger.error("--search requires --query");
+      process.exit(1);
+    }
+    logger.log(`Searching: "${query}" (max ${maxResults} results)`);
+    const result = await search(query, { maxResults, debug });
+    console.log("\n" + result);
+    return;
+  }
+  if (url !== "") {
+    if (query === "") {
+      logger.error("--url requires --query");
+      process.exit(1);
+    }
+    await processUrl(url, query, debug);
+    return;
+  }
+  await processFile(filePath, query, debug);
+}
+async function processUrl(url, query, debug) {
+  logger.log(`Fetching: ${url}`);
+  logger.log(`Query: "${query}"`);
+  logger.log(`Query tokens: ${JSON.stringify(tokenize(query))}`);
+  try {
+    const response = await fetch(url, {
+      headers: {
+        "User-Agent": "Mozilla/5.0 (compatible; PeekyBot/1.0)",
+        "Accept": "text/html"
+      }
+    });
+    if (!response.ok) {
+      logger.error(`HTTP ${response.status}: ${response.statusText}`);
+      process.exit(1);
+    }
+    const html = await response.text();
+    processHtml(html, query, debug);
+  } catch (err) {
+    logger.error(`Fetch failed: ${err}`);
+    process.exit(1);
+  }
+}
+async function processFile(filePath, query, debug) {
+  if (filePath === "") {
+    filePath = path.join(__dirname, "../samples", "clerk-how-to-add-user-to-org-sample.html");
+  }
+  if (query === "") {
+    query = "how to add user to organization";
+  }
+  if (!fs.existsSync(filePath)) {
+    logger.error(`File not found: ${filePath}`);
+    process.exit(1);
+  }
+  logger.log(`Reading file: ${filePath}`);
+  logger.log(`Query: "${query}"`);
+  logger.log(`Query tokens: ${JSON.stringify(tokenize(query))}`);
+  const html = fs.readFileSync(filePath, "utf8");
+  processHtml(html, query, debug);
+}
+function processHtml(html, query, debug) {
+  const config = {
+    debug,
+    excerpts: {
+      maxExcerpts: 3,
+      charBudget: 4e3
+    }
+  };
+  const result = extractExcerpts(html, query, config);
+  logger.printTimings();
+  console.log("\n" + "=".repeat(60));
+  console.log(formatExcerpts(result));
+  if (debug && "debug" in result && result.debug) {
+    logger.debug("=".repeat(60));
+    logger.debug("Debug Info:");
+    logger.debug(`  Sentences: ${result.debug.sentenceCount}`);
+    logger.debug(`  Query term coverage: ${((result.debug.queryTermCoverage ?? 0) * 100).toFixed(1)}%`);
+    logger.debug(`  Max raw BM25: ${(result.debug.maxRawBm25 ?? 0).toFixed(3)}`);
+    logger.debug(`  Has relevant results: ${result.debug.hasRelevantResults ?? "N/A"}`);
+    logger.debug(`  Anchors selected: ${result.debug.anchorCount}`);
+    logger.debug(`  Chunks before dedupe: ${result.debug.chunkCount}`);
+    logger.debug(`  Chunks after dedupe: ${result.debug.dedupedChunkCount}`);
+    logger.debug("\nTop 10 sentences by score:");
+    for (const s of result.debug.topSentences) {
+      const pathStr = s.headingPath.length > 0 ? ` [${s.headingPath.join(" > ")}]` : "";
+      logger.debug(`  ${s.score.toFixed(3)}${pathStr}: ${s.text}`);
+    }
+  }
+  console.log("=".repeat(60));
+}
+function printUsage() {
+  console.log(`
+peeky-search: IR-based excerpt extraction from HTML
+
+Usage:
+  node dist/index.js [options]
+
+Modes:
+  --file path       Process a local HTML file (default mode)
+  --url URL         Fetch and process a single URL
+  --search          Search via SearXNG and extract from multiple pages
+
+Options:
+  --query "text"    Search query (required for --url and --search)
+  --max N           Max results for --search mode (default: 5)
+  --debug           Show debug information
+  --timing, -t      Show performance timing breakdown
+  --help, -h        Show this help message
+
+Examples:
+  # Process local file
+  node dist/index.js --query "invite member" --file page.html
+
+  # Fetch and process a URL
+  node dist/index.js --url "https://clerk.com/docs/..." --query "invitation redirect"
+
+  # Search via SearXNG (requires SEARXNG_URL env var)
+  node dist/index.js --search --query "Clerk organization invitation redirect_url"
+
+  # Run with timing enabled
+  node dist/index.js --search --query "React hooks" --timing
+`);
+}
+main().catch((err) => {
+  logger.error(`Unexpected error: ${err}`);
+  process.exit(1);
+});

package/dist/mcp/server.js

@@ -0,0 +1,67 @@
+import {
+  logger_default,
+  search
+} from "../chunk-S3WZDJCP.js";
+
+// src/mcp/server.ts
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { z } from "zod";
+var logger = logger_default.getInstance();
+var server = new McpServer({
+  name: "peeky_mcp",
+  version: "1.0.0"
+});
+server.tool(
+  "peeky_web_search",
+  `Search the web for technical information. Use this tool for ANY request involving research, documentation lookup, or questions you're not 100% confident about.
+
+WHEN TO USE (prefer this over browsing or guessing):
+- User says "research", "look up", "find docs", "search for"
+- User asks "how do I...", "what is...", "best way to..."
+- User asks about libraries, frameworks, APIs, tools
+- User has an error message or needs debugging help
+- You need current/accurate information beyond your training data
+
+QUERY FORMULATION:
+- For vague or project-level requests, extract the underlying technical concepts and search for those
+- Always include the specific technology/framework/library name
+- Focus on patterns, APIs, or techniques that have documentation - not the user's exact project
+- If a topic is niche, think about what general concept it falls under
+
+SEARCH OPERATORS:
+- site:domain.com - limit to specific domain
+- "exact phrase" - exact phrase match
+- -term - exclude term
+
+RETURNS: Extracted text excerpts with source URLs (not raw HTML).`,
+  {
+    query: z.string().describe(
+      'Search query with technical terms. Supports operators: site:, "quotes", -exclude.'
+    ),
+    maxResults: z.number().optional().describe("Maximum pages to scrape (default: 5, max: 10)"),
+    diagnostics: z.boolean().optional().describe("Include detailed diagnostics about why pages were filtered or failed (default: false). Diagnostics are always shown when no results are found.")
+  },
+  async ({ query, maxResults, diagnostics }) => {
+    const result = await search(query, {
+      ...maxResults !== void 0 && { maxResults },
+      ...diagnostics !== void 0 && { diagnostics }
+    });
+    return {
+      content: [
+        {
+          type: "text",
+          text: result
+        }
+      ]
+    };
+  }
+);
+async function main() {
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+}
+main().catch((error) => {
+  logger.error(`Fatal error: ${error}`);
+  process.exit(1);
+});

package/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "peeky-search",
+  "version": "1.0.10",
+  "description": "IR-based HTML content extraction with MCP server for web search",
+  "type": "module",
+  "main": "dist/index.js",
+  "bin": {
+    "peeky-search": "dist/cli.js"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsup",
+    "build:watch": "tsup --watch",
+    "build:tsc": "tsc --project tsconfig.json",
+    "start": "node dist/cli.js",
+    "start:watch": "nodemon --watch src --ext *.ts --exec \"pnpm build && pnpm start\"",
+    "mcp": "node dist/cli.js mcp",
+    "cli": "node dist/cli.js"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT",
+  "packageManager": "pnpm@10.14.0",
+  "devDependencies": {
+    "@types/node": "^22.19.7",
+    "tsup": "^8.5.1",
+    "typescript": "^5.9.3"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.0.0",
+    "cheerio": "^1.2.0",
+    "domhandler": "^5.0.3",
+    "stemmer": "^2.0.1",
+    "zod": "^3.24.0"
+  }
+}