skdev-websearch-mcp 1.0.0__py3-none-any.whl

arxiv.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET
+from typing import Any
+
+from config import get_http_client
+
+
+async def search_arxiv(query: str, count: int = 3, search_field: str = "all",
+                       sort_by: str = "relevance", sort_order: str = "descending",
+                       start: int = 0, id_list: str = "",
+                       category: str = "", raw_query: str = "") -> dict:
+    try:
+        if raw_query:
+            params = {"search_query": raw_query, "start": start,
+                "max_results": min(count, 50), "sortBy": sort_by, "sortOrder": sort_order}
+        elif id_list:
+            params = {"id_list": id_list, "start": start,
+                "max_results": min(count, 50), "sortBy": sort_by, "sortOrder": sort_order}
+        else:
+            search_q = f"{search_field}:{query}" if search_field != "all" else f"all:{query}"
+            if category:
+                search_q += f" AND cat:{category}"
+            params = {"search_query": search_q, "start": start,
+                "max_results": min(count, 50), "sortBy": sort_by, "sortOrder": sort_order}
+        c = get_http_client()
+        r = await c.get("https://export.arxiv.org/api/query", params=params,
+                        headers={"User-Agent": "mcp-codesearch/1.0"}, timeout=15)
+        if r.status_code != 200:
+            return {"results": [], "total_available": 0}
+        ns = {"atom": "http://www.w3.org/2005/Atom",
+              "arxiv": "http://arxiv.org/schemas/atom"}
+        root = ET.fromstring(r.text)
+        total = root.findtext("opensearch:totalResults", "0",
+                              {"opensearch": "http://a9.com/-/spec/opensearch/1.1/"})
+        results = []
+        for entry in root.findall("atom:entry", ns)[:count]:
+            title = entry.findtext("atom:title", "", ns).replace("\n", " ").strip()
+            summary_text = entry.findtext("atom:summary", "", ns).replace("\n", " ").strip()[:500]
+            url = ""
+            for link in entry.findall("atom:link", ns):
+                if link.get("title") == "pdf":
+                    url = link.get("href", "")
+                    break
+                if link.get("rel") == "alternate":
+                    url = link.get("href", "")
+            authors = [a.findtext("atom:name", "", ns) for a in entry.findall("atom:author", ns)]
+            published = entry.findtext("atom:published", "", ns)[:10]
+            updated = entry.findtext("atom:updated", "", ns)[:10]
+            categories = [c.get("term", "") for c in entry.findall("atom:category", ns)]
+            comment = entry.findtext("arxiv:comment", "", ns)
+            journal_ref = entry.findtext("arxiv:journal_ref", "", ns)
+            doi = entry.findtext("arxiv:doi", "", ns)
+            pc = entry.find("arxiv:primary_category", ns)
+            primary_cat = pc.get("term", "") if pc is not None else ""
+            id_text = entry.findtext("atom:id", "", ns)
+            arxiv_id = id_text.split("/abs/")[-1] if "/abs/" in id_text else ""
+            results.append({
+                "title": title, "url": url, "snippet": summary_text,
+                "authors": authors, "published": published, "updated": updated,
+                "source": "arxiv", "arxiv_id": arxiv_id,
+                "categories": categories, "primary_category": primary_cat,
+                "comment": comment, "journal_ref": journal_ref, "doi": doi,
+            })
+        return {"results": results, "total_available": int(total)}
+    except Exception:
+        return {"results": [], "total_available": 0}

bench_prof.py

@@ -0,0 +1,77 @@
+"""Detailed end-to-end benchmark of websearch search_multi with all engine keys loaded.
+
+Diagnostic only. Loads env from the server's `run` file via bash (so ${VAR:-default}
+expands exactly as the server sees), instruments every engine + phase on the `research`
+namespace, runs search_multi once, prints a per-phase bill.
+"""
+import asyncio, os, subprocess, sys, time, tempfile
+
+WSDIR = "/home/sk/session-root/repos/websearch"
+
+res = subprocess.run(["bash", "-c",
+    f"source <(grep -E '^export ' {WSDIR}/run); env | grep -E '^(GROQ|TAVILY|TINYFISH|ANYSEARCH|NV|RERANKER)_'"],
+    capture_output=True, text=True)
+for line in res.stdout.splitlines():
+    if "=" in line:
+        k, _, v = line.partition("=")
+        os.environ[k] = v
+
+sys.path.insert(0, WSDIR)
+import research
+
+from collections import defaultdict
+TIMES = defaultdict(float)
+COUNTS = defaultdict(int)
+RESULTS = defaultdict(int)
+_t0 = {}
+
+def _timeit(name):
+    def deco(fn):
+        async def wrap(*a, **k):
+            key = name
+            if key not in _t0:
+                _t0[key] = time.monotonic()
+            r = await fn(*a, **k)
+            dt = time.monotonic() - _t0.pop(key, time.monotonic())
+            TIMES[key] += dt
+            COUNTS[key] += 1
+            if isinstance(r, dict):
+                RESULTS[key] += len(r.get("results", []))
+            elif isinstance(r, list):
+                RESULTS[key] += len(r)
+            return r
+        return wrap
+    return deco
+
+_existing = {}
+def patch(mod, name):
+    _existing[name] = getattr(mod, name)
+    setattr(mod, name, _timeit(name)(_existing[name]))
+
+# Engine phases in research namespace
+for fn in ["search_google_rss", "search_tavily", "search_wikipedia", "search_reddit",
+           "search_arxiv", "search_anysearch", "tinyfish_search", "search_ddg",
+           "search_brave"]:
+    if hasattr(research, fn):
+        patch(research, fn)
+# Groq + rerank phases
+for fn in ["rewrite_query", "classify_need", "decompose_query", "expand_query", "_rerank"]:
+    if hasattr(research, fn):
+        patch(research, fn)
+
+query = sys.argv[1] if len(sys.argv) > 1 else "best laptops 2026 under 1000 dollars"
+count = int(sys.argv[2]) if len(sys.argv) > 2 else 50
+
+async def main():
+    t0 = time.monotonic()
+    from config import TAVILY_KEYS, TINYFISH_KEYS, GROQ_API_KEYS, NV_KEY
+    print(f"keys: tavily={len(TAVILY_KEYS)} tinyfish={len(TINYFISH_KEYS)} groq={len(GROQ_API_KEYS)} nv={'y' if NV_KEY else 'n'}", flush=True)
+    r = await research.search_multi(query, count=count, tavily_depth="advanced")
+    twall = time.monotonic() - t0
+    print(f"\n=== SEARCH_MULTI  q={query!r} count={count} ===")
+    print(f"{'phase':<22}{'sec':>8}{'calls':>6}{'res':>6}")
+    for k in sorted(TIMES, key=lambda x: -TIMES[x]):
+        print(f"{k:<22}{TIMES[k]:>8.2f}{COUNTS[k]:>6}{RESULTS[k]:>6}")
+    print(f"\nTOTAL WALL: {twall:.1f}s  results={len(r.get('results', []))}")
+
+asyncio.run(main())