searchpin 1.0.0__tar.gz

searchpin-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025-2026 Searchpin Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

searchpin-1.0.0/MANIFEST.in

@@ -0,0 +1,4 @@
+include README.md
+include LICENSE
+include requirements.txt
+recursive-include searchpin *.py

searchpin-1.0.0/PKG-INFO

@@ -0,0 +1,113 @@
+Metadata-Version: 2.4
+Name: searchpin
+Version: 1.0.0
+Summary: Self-hosted web search for AI agents — zero API keys, embedding re-rank, multi-engine parallel search
+Author: Searchpin Contributors
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/telly6/searchpin
+Project-URL: Repository, https://github.com/telly6/searchpin
+Project-URL: Issues, https://github.com/telly6/searchpin/issues
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: fastembed>=0.3.0
+Requires-Dist: numpy
+Requires-Dist: brotli
+Dynamic: license-file
+
+# Searchpin
+
+[![PyPI version](https://img.shields.io/pypi/v/searchpin)](https://pypi.org/project/searchpin/)
+[![Python](https://img.shields.io/pypi/pyversions/searchpin)](https://pypi.org/project/searchpin/)
+[![License](https://img.shields.io/pypi/l/searchpin)](LICENSE)
+[![Docker](https://img.shields.io/badge/docker-ghcr.io-blue)](https://github.com/telly6/searchpin/pkgs/container/searchpin)
+
+Self-hosted web search for AI agents — zero API keys, zero cost. `pip install searchpin` and you're done.
+
+- **Search quality rivals commercial products** — four search engines in parallel (Baidu, Sogou, Bing CN, Bing Intl), merged and re-ranked by semantic similarity. A rare capability among free MCP search tools.
+- **Zero-cost agent development** — no API keys, no sign-ups, no usage limits. Pair with a local LLM and your entire development loop costs nothing. Run 24/7 agent experiments without worrying about quotas.
+- **Pollution detection** — automatically flags results that are unrelated to your query, so your agent doesn't chase irrelevant content.
+- **Cross-verification** — results from four independent sources let your LLM corroborate information across engines, raising the credibility of what it finds.
+- **Content extraction that handles modern sites** — goes beyond basic HTML-to-text to extract SSR hydration payloads (Next.js, Nuxt), JSON-LD structured data, and microdata from pages that would otherwise return empty.
+- **Token-conscious output** — results are titles, URLs, and snippets only. Your LLM decides which pages are worth fetching in full. Structured extraction data is compact and truncated, keeping token overhead under control.
+
+## Quick Start
+
+```bash
+pip install searchpin
+searchpin-server
+```
+
+## Configuration
+
+### Claude Desktop / Cursor / any MCP client
+
+Add to your `mcpServers` config:
+
+```json
+{
+  "mcpServers": {
+    "Searchpin": {
+      "command": "searchpin-server",
+      "args": []
+    }
+  }
+}
+```
+
+### VS Code
+
+[![Install in VS Code](https://img.shields.io/badge/VS_Code-Install-0098FF?style=flat-square&logo=visualstudiocode&logoColor=white)](https://insiders.vscode.dev/redirect/mcp/install?name=Searchpin&config=%7B%22command%22%3A%22searchpin-server%22%2C%22args%22%3A%5B%5D%7D)
+[![Install in VS Code Insiders](https://img.shields.io/badge/VS_Code_Insiders-Install-24bfa5?style=flat-square&logo=visualstudiocode&logoColor=white)](https://insiders.vscode.dev/redirect/mcp/install?name=Searchpin&config=%7B%22command%22%3A%22searchpin-server%22%2C%22args%22%3A%5B%5D%7D&quality=insiders)
+
+Or manually, add to `.vscode/mcp.json`:
+
+```json
+{
+  "servers": {
+    "Searchpin": {
+      "command": "searchpin-server",
+      "args": []
+    }
+  }
+}
+```
+
+### Docker
+
+```bash
+docker run -i --rm ghcr.io/telly6/searchpin:latest
+```
+
+```json
+{
+  "mcpServers": {
+    "Searchpin": {
+      "command": "docker",
+      "args": ["run", "-i", "--rm", "ghcr.io/telly6/searchpin:latest"]
+    }
+  }
+}
+```
+
+### Python API
+
+```python
+from searchpin import SearchEngine
+
+engine = SearchEngine()
+results = engine.search("Python 3.13 new features")
+page = engine.fetch("https://docs.python.org/3/whatsnew/3.13.html")
+engine.close()
+```
+
+
+

searchpin-1.0.0/README.md

@@ -0,0 +1,88 @@
+# Searchpin
+
+[![PyPI version](https://img.shields.io/pypi/v/searchpin)](https://pypi.org/project/searchpin/)
+[![Python](https://img.shields.io/pypi/pyversions/searchpin)](https://pypi.org/project/searchpin/)
+[![License](https://img.shields.io/pypi/l/searchpin)](LICENSE)
+[![Docker](https://img.shields.io/badge/docker-ghcr.io-blue)](https://github.com/telly6/searchpin/pkgs/container/searchpin)
+
+Self-hosted web search for AI agents — zero API keys, zero cost. `pip install searchpin` and you're done.
+
+- **Search quality rivals commercial products** — four search engines in parallel (Baidu, Sogou, Bing CN, Bing Intl), merged and re-ranked by semantic similarity. A rare capability among free MCP search tools.
+- **Zero-cost agent development** — no API keys, no sign-ups, no usage limits. Pair with a local LLM and your entire development loop costs nothing. Run 24/7 agent experiments without worrying about quotas.
+- **Pollution detection** — automatically flags results that are unrelated to your query, so your agent doesn't chase irrelevant content.
+- **Cross-verification** — results from four independent sources let your LLM corroborate information across engines, raising the credibility of what it finds.
+- **Content extraction that handles modern sites** — goes beyond basic HTML-to-text to extract SSR hydration payloads (Next.js, Nuxt), JSON-LD structured data, and microdata from pages that would otherwise return empty.
+- **Token-conscious output** — results are titles, URLs, and snippets only. Your LLM decides which pages are worth fetching in full. Structured extraction data is compact and truncated, keeping token overhead under control.
+
+## Quick Start
+
+```bash
+pip install searchpin
+searchpin-server
+```
+
+## Configuration
+
+### Claude Desktop / Cursor / any MCP client
+
+Add to your `mcpServers` config:
+
+```json
+{
+  "mcpServers": {
+    "Searchpin": {
+      "command": "searchpin-server",
+      "args": []
+    }
+  }
+}
+```
+
+### VS Code
+
+[![Install in VS Code](https://img.shields.io/badge/VS_Code-Install-0098FF?style=flat-square&logo=visualstudiocode&logoColor=white)](https://insiders.vscode.dev/redirect/mcp/install?name=Searchpin&config=%7B%22command%22%3A%22searchpin-server%22%2C%22args%22%3A%5B%5D%7D)
+[![Install in VS Code Insiders](https://img.shields.io/badge/VS_Code_Insiders-Install-24bfa5?style=flat-square&logo=visualstudiocode&logoColor=white)](https://insiders.vscode.dev/redirect/mcp/install?name=Searchpin&config=%7B%22command%22%3A%22searchpin-server%22%2C%22args%22%3A%5B%5D%7D&quality=insiders)
+
+Or manually, add to `.vscode/mcp.json`:
+
+```json
+{
+  "servers": {
+    "Searchpin": {
+      "command": "searchpin-server",
+      "args": []
+    }
+  }
+}
+```
+
+### Docker
+
+```bash
+docker run -i --rm ghcr.io/telly6/searchpin:latest
+```
+
+```json
+{
+  "mcpServers": {
+    "Searchpin": {
+      "command": "docker",
+      "args": ["run", "-i", "--rm", "ghcr.io/telly6/searchpin:latest"]
+    }
+  }
+}
+```
+
+### Python API
+
+```python
+from searchpin import SearchEngine
+
+engine = SearchEngine()
+results = engine.search("Python 3.13 new features")
+page = engine.fetch("https://docs.python.org/3/whatsnew/3.13.html")
+engine.close()
+```
+
+
+

searchpin-1.0.0/pyproject.toml

@@ -0,0 +1,68 @@
+[build-system]
+requires = ["setuptools>=64", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "searchpin"
+version = "1.0.0"
+description = "Self-hosted web search for AI agents — zero API keys, embedding re-rank, multi-engine parallel search"
+authors = [
+    {name = "Searchpin Contributors"}
+]
+license = "MIT"
+readme = "README.md"
+requires-python = ">=3.10"
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "fastembed>=0.3.0",
+    "numpy",
+    "brotli",
+]
+
+[project.scripts]
+searchpin-server = "search_server:main"
+
+[project.urls]
+Homepage = "https://github.com/telly6/searchpin"
+Repository = "https://github.com/telly6/searchpin"
+Issues = "https://github.com/telly6/searchpin/issues"
+
+[tool.setuptools.packages.find]
+include = ["searchpin", "searchpin.*"]
+
+[tool.setuptools]
+py-modules = ["search_server"]
+
+# ── pytest ──────────────────────────────────────────────────
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+
+# ── ruff (lint + format) ────────────────────────────────────
+[tool.ruff]
+target-version = "py310"
+line-length = 120
+exclude = ["dev/", "proxy.py", ".venv/", "__pycache__/"]
+
+[tool.ruff.lint]
+select = [
+    "E",   # pycodestyle errors
+    "F",   # pyflakes
+    "W",   # pycodestyle warnings
+    "I",   # isort
+    "UP",  # pyupgrade
+]
+ignore = ["E501"]  # line length — logged f-strings are intentional
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"

searchpin-1.0.0/requirements.txt

@@ -0,0 +1,3 @@
+fastembed>=0.3.0
+numpy
+brotli

searchpin-1.0.0/search_server.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+"""
+Searchpin MCP Server (stdio transport)
+AI agent launches this as a subprocess. Reads JSON-RPC from stdin, writes to stdout.
+"""
+
+import json
+import sys
+
+from searchpin.config import DEFAULT_MODEL_NAME, PRODUCT_NAME
+from searchpin.engine import MCP_TOOLS, SearchEngine
+
+
+def build_response(rid, result):
+    return {"jsonrpc": "2.0", "id": rid, "result": result}
+
+
+def build_error(rid, code, message):
+    return {"jsonrpc": "2.0", "id": rid, "error": {"code": code, "message": message}}
+
+
+def handle_mcp_request(body, engine):
+    rid = body.get("id")
+    method = body.get("method", "")
+    params = body.get("params", {})
+
+    if method == "initialize":
+        return build_response(
+            rid,
+            {
+                "protocolVersion": "2024-11-05",
+                "capabilities": {"tools": {}},
+                "serverInfo": {"name": PRODUCT_NAME, "version": "1.0.0"},
+            },
+        )
+    elif method == "notifications/initialized":
+        return None
+    elif method == "ping":
+        return build_response(rid, {})
+    elif method == "tools/list":
+        return build_response(rid, {"tools": MCP_TOOLS})
+    elif method == "resources/list":
+        return build_response(rid, {"resources": []})
+    elif method == "prompts/list":
+        return build_response(rid, {"prompts": []})
+    elif method == "tools/call":
+        tool_name = params.get("name", "")
+        args = params.get("arguments", {})
+        if tool_name == "web_search":
+            print(
+                f"[search_server] args keys={list(args.keys())!r} "
+                f"topic={args.get('topic')!r} "
+                f"exclude_domains={args.get('exclude_domains')!r}",
+                file=sys.stderr,
+                flush=True,
+            )
+            result = engine.search(
+                args.get("query", ""),
+                args.get("max_results", 10),
+                args.get("freshness"),
+                topic=args.get("topic"),
+                exclude_domains=args.get("exclude_domains"),
+                include_domains=args.get("include_domains"),
+            )
+        elif tool_name == "web_fetch":
+            result = engine.fetch(args.get("url", ""))
+        else:
+            return build_error(rid, -32601, f"Unknown tool: {tool_name}")
+        return build_response(
+            rid,
+            {
+                "content": [{"type": "text", "text": json.dumps(result, ensure_ascii=False, indent=2)}],
+            },
+        )
+    else:
+        return build_error(rid, -32601, f"Method not found: {method}")
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Searchpin MCP Server")
+    parser.add_argument(
+        "--model",
+        type=str,
+        default=None,
+        help="Embedding model name (default: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2)",
+    )
+    args = parser.parse_args()
+
+    print(f"[{PRODUCT_NAME}] starting engine (stdio mode)...", file=sys.stderr, flush=True)
+
+    engine = SearchEngine(
+        model_name=args.model or DEFAULT_MODEL_NAME,
+        max_workers=3,
+    )
+
+    print(f"[{PRODUCT_NAME}] engine ready, waiting for requests on stdin", file=sys.stderr, flush=True)
+
+    for line in sys.stdin:
+        line = line.strip()
+        if not line:
+            continue
+
+        try:
+            body = json.loads(line)
+            result = handle_mcp_request(body, engine)
+            if result is not None:
+                sys.stdout.write(json.dumps(result, ensure_ascii=False) + "\n")
+                sys.stdout.flush()
+        except json.JSONDecodeError as e:
+            err = build_error(None, -32700, f"Parse error: {e}")
+            sys.stdout.write(json.dumps(err, ensure_ascii=False) + "\n")
+            sys.stdout.flush()
+        except Exception as e:
+            print(f"[{PRODUCT_NAME}] error: {e}", file=sys.stderr, flush=True)
+            err = build_error(None, -32603, str(e))
+            sys.stdout.write(json.dumps(err, ensure_ascii=False) + "\n")
+            sys.stdout.flush()
+
+    engine.close()
+
+
+if __name__ == "__main__":
+    main()

searchpin-1.0.0/searchpin/__init__.py

@@ -0,0 +1,17 @@
+"""
+Searchpin — Self-hosted web search for AI agents.
+
+Usage:
+    from searchpin import SearchEngine
+
+    engine = SearchEngine()
+    results = engine.search("your query")
+    page = engine.fetch("https://example.com/article")
+    engine.close()
+"""
+
+from searchpin.config import DEFAULT_MODEL_NAME, PRODUCT_NAME
+from searchpin.engine import MCP_TOOLS, SearchEngine
+
+__version__ = "1.0.0"
+__all__ = ["SearchEngine", "MCP_TOOLS", "PRODUCT_NAME", "DEFAULT_MODEL_NAME"]

searchpin-1.0.0/searchpin/__main__.py

@@ -0,0 +1,11 @@
+"""Entry point for python -m searchpin — starts the MCP stdio server.
+
+Usage:
+    python -m searchpin              # Start with defaults
+    python -m searchpin --model ...  # Custom embedding model
+"""
+
+from search_server import main
+
+if __name__ == "__main__":
+    main()