oxe 0.1.0__py3-none-any.whl

oxe/__init__.py

@@ -0,0 +1,3 @@
+"""oxe: Exa-compatible web-search proxy and MCP server backed by DuckDuckGo."""
+
+__version__ = "0.1.0"

oxe/__main__.py

@@ -0,0 +1,19 @@
+import os
+
+import uvicorn
+
+from .server import app
+
+
+def main() -> None:
+    uvicorn.run(
+        app,
+        host="127.0.0.1",
+        port=int(os.getenv("OXE_PORT", "4479")),
+        workers=1,
+        log_level=os.getenv("OXE_LOG_LEVEL", "info").lower(),
+    )
+
+
+if __name__ == "__main__":
+    main()

oxe/cache.py

@@ -0,0 +1,245 @@
+import gzip
+import json
+import os
+import sqlite3
+import threading
+import time
+from pathlib import Path
+
+SCHEMA = """
+PRAGMA journal_mode = WAL;
+PRAGMA synchronous = NORMAL;
+PRAGMA cache_size = -32000;
+PRAGMA temp_store = MEMORY;
+PRAGMA busy_timeout = 5000;
+
+CREATE TABLE IF NOT EXISTS cache (
+  query_hash  TEXT PRIMARY KEY,
+  query_text  TEXT NOT NULL,
+  response    BLOB NOT NULL,
+  expires_at  INTEGER NOT NULL,
+  hits        INTEGER NOT NULL DEFAULT 0
+) WITHOUT ROWID;
+CREATE INDEX IF NOT EXISTS expires_idx ON cache(expires_at);
+
+CREATE TABLE IF NOT EXISTS clicks (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  query_hash TEXT    NOT NULL,
+  result_id  TEXT    NOT NULL,
+  url        TEXT    NOT NULL,
+  title      TEXT    NOT NULL,
+  clicked_at INTEGER NOT NULL,
+  source     TEXT    NOT NULL DEFAULT 'web'
+);
+CREATE INDEX IF NOT EXISTS clicks_query_idx  ON clicks(query_hash);
+CREATE INDEX IF NOT EXISTS clicks_recent_idx ON clicks(clicked_at);
+"""
+
+
+class TTLCache:
+    def __init__(self, db_path: str | os.PathLike[str]):
+        self.db_path = str(db_path)
+        Path(self.db_path).parent.mkdir(parents=True, exist_ok=True)
+        self._lock = threading.Lock()
+        self._conn = sqlite3.connect(self.db_path, check_same_thread=False, isolation_level=None)
+        self._conn.executescript(SCHEMA)
+        self._conn.execute("PRAGMA optimize;")
+
+    def get(self, key: str) -> dict | None:
+        now = int(time.time())
+        with self._lock:
+            row = self._conn.execute(
+                "SELECT response, expires_at FROM cache WHERE query_hash = ?", (key,)
+            ).fetchone()
+        if row is None:
+            return None
+        blob, expires_at = row
+        if expires_at < now:
+            return None
+        try:
+            payload = gzip.decompress(blob)
+        except (OSError, gzip.BadGzipFile):
+            return None
+        with self._lock:
+            self._conn.execute(
+                "UPDATE cache SET hits = hits + 1 WHERE query_hash = ?", (key,)
+            )
+        return json.loads(payload)
+
+    def set(self, key: str, value: dict, ttl: int) -> None:
+        payload = gzip.compress(json.dumps(value, separators=(",", ":")).encode("utf-8"))
+        expires_at = int(time.time()) + ttl
+        with self._lock:
+            self._conn.execute(
+                "INSERT OR REPLACE INTO cache (query_hash, query_text, response, expires_at, hits) "
+                "VALUES (?, ?, ?, ?, COALESCE((SELECT hits FROM cache WHERE query_hash = ?), 0))",
+                (key, value.get("_q", ""), payload, expires_at, key),
+            )
+            self._conn.execute("DELETE FROM cache WHERE expires_at < ?", (expires_at - ttl - 1,))
+
+    def invalidate(self) -> int:
+        with self._lock:
+            cur = self._conn.execute("DELETE FROM cache")
+            return cur.rowcount
+
+    def list_rows(
+        self,
+        q: str | None = None,
+        include_expired: bool = False,
+        limit: int = 100,
+        offset: int = 0,
+    ) -> list[dict]:
+        now = int(time.time())
+        clauses: list[str] = []
+        params: list = []
+        if not include_expired:
+            clauses.append("expires_at >= ?")
+            params.append(now)
+        if q:
+            clauses.append("query_text LIKE ?")
+            params.append(f"%{q}%")
+        where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
+        sql = (
+            "SELECT query_hash, query_text, expires_at, hits, length(response) "
+            f"FROM cache{where} ORDER BY expires_at DESC LIMIT ? OFFSET ?"
+        )
+        params.extend([limit, offset])
+        with self._lock:
+            rows = self._conn.execute(sql, params).fetchall()
+        return [
+            {
+                "hash": h,
+                "query": qtext,
+                "expires_at": exp,
+                "hits": hits,
+                "size_bytes": size,
+                "expired": exp < now,
+            }
+            for h, qtext, exp, hits, size in rows
+        ]
+
+    def peek(self, key: str) -> dict | None:
+        return self.get(key)
+
+    def delete(self, key: str) -> bool:
+        with self._lock:
+            cur = self._conn.execute("DELETE FROM cache WHERE query_hash = ?", (key,))
+            return cur.rowcount > 0
+
+    def stats(self) -> dict:
+        now = int(time.time())
+        with self._lock:
+            total = self._conn.execute("SELECT COUNT(*) FROM cache").fetchone()[0]
+            unexpired = self._conn.execute(
+                "SELECT COUNT(*) FROM cache WHERE expires_at >= ?", (now,)
+            ).fetchone()[0]
+            total_hits = self._conn.execute("SELECT COALESCE(SUM(hits), 0) FROM cache").fetchone()[0]
+            oldest = self._conn.execute(
+                "SELECT MIN(expires_at) FROM cache WHERE expires_at >= ?", (now,)
+            ).fetchone()[0]
+            newest_row = self._conn.execute(
+                "SELECT MAX(expires_at) FROM cache"
+            ).fetchone()[0]
+        db_size = os.path.getsize(self.db_path) if os.path.exists(self.db_path) else 0
+        return {
+            "rows": total,
+            "unexpired_rows": unexpired,
+            "db_size_bytes": db_size,
+            "total_hits": total_hits,
+            "oldest_unexpired": oldest,
+            "newest": newest_row,
+        }
+
+    def close(self) -> None:
+        with self._lock:
+            self._conn.close()
+
+    # -- click tracking -----------------------------------------------------
+
+    def record_click(
+        self,
+        query_hash: str,
+        result_id: str,
+        url: str,
+        title: str,
+        source: str = "web",
+    ) -> int:
+        now = int(time.time())
+        with self._lock:
+            cur = self._conn.execute(
+                "INSERT INTO clicks (query_hash, result_id, url, title, clicked_at, source) "
+                "VALUES (?, ?, ?, ?, ?, ?)",
+                (query_hash, result_id, url, title, now, source),
+            )
+            return cur.lastrowid or 0
+
+    def get_clicks(
+        self,
+        query_hash: str | None = None,
+        query_text: str | None = None,
+        limit: int = 50,
+        since_hours: int | None = None,
+    ) -> list[dict]:
+        clauses: list[str] = []
+        params: list = []
+        join = " LEFT JOIN cache k ON k.query_hash = c.query_hash "
+        if query_hash:
+            clauses.append("c.query_hash = ?")
+            params.append(query_hash)
+        if query_text:
+            clauses.append("k.query_text LIKE ?")
+            params.append(f"%{query_text}%")
+        if since_hours is not None:
+            clauses.append("c.clicked_at >= ?")
+            params.append(int(time.time()) - since_hours * 3600)
+        where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
+        sql = (
+            "SELECT c.id, c.query_hash, COALESCE(k.query_text, ''), c.result_id, c.url, "
+            "c.title, c.clicked_at, c.source "
+            f"FROM clicks c{join}{where} ORDER BY c.clicked_at DESC LIMIT ?"
+        )
+        params.append(limit)
+        with self._lock:
+            rows = self._conn.execute(sql, params).fetchall()
+        return [
+            {
+                "id": cid,
+                "query_hash": qh,
+                "query": qt,
+                "result_id": rid,
+                "url": url,
+                "title": title,
+                "clicked_at": ts,
+                "source": src,
+            }
+            for cid, qh, qt, rid, url, title, ts, src in rows
+        ]
+
+    def click_stats(self) -> dict:
+        now = int(time.time())
+        with self._lock:
+            total = self._conn.execute("SELECT COUNT(*) FROM clicks").fetchone()[0]
+            last_24h = self._conn.execute(
+                "SELECT COUNT(*) FROM clicks WHERE clicked_at >= ?", (now - 86400,)
+            ).fetchone()[0]
+            oldest = self._conn.execute("SELECT MIN(clicked_at) FROM clicks").fetchone()[0]
+        return {"total": total, "last_24h": last_24h, "oldest": oldest}
+
+    def prune_clicks(self, retention_days: int) -> int:
+        cutoff = int(time.time()) - retention_days * 86400
+        with self._lock:
+            cur = self._conn.execute("DELETE FROM clicks WHERE clicked_at < ?", (cutoff,))
+            return cur.rowcount
+
+    def delete_clicks(self, scope: str) -> int:
+        """scope: '24h' deletes last 24h, 'all' deletes everything."""
+        with self._lock:
+            if scope == "all":
+                cur = self._conn.execute("DELETE FROM clicks")
+            elif scope == "24h":
+                cur = self._conn.execute(
+                    "DELETE FROM clicks WHERE clicked_at >= ?", (int(time.time()) - 86400,)
+                )
+            else:
+                return 0
+            return cur.rowcount

oxe/exa_compat.py

@@ -0,0 +1,131 @@
+import hashlib
+import logging
+import re
+import uuid
+from typing import Any
+from urllib.parse import urlparse
+
+from ddgs import DDGS
+
+log = logging.getLogger(__name__)
+
+_SENTENCE_SPLIT = re.compile(r"(?<=[.!?])\s+")
+_NUM_CLAMP = (1, 30)
+_FAVICON = "https://www.google.com/s2/favicons?domain={netloc}&sz=32"
+
+
+def cache_key(req: dict) -> str:
+    contents = req.get("contents") or {}
+    norm = (
+        (req.get("query") or "").lower().strip(),
+        max(_NUM_CLAMP[0], min(_NUM_CLAMP[1], int(req.get("numResults") or 10))),
+        req.get("type") or "auto",
+        tuple(sorted(req.get("includeDomains") or [])),
+        tuple(sorted(req.get("excludeDomains") or [])),
+        bool(contents.get("highlights")),
+        bool(contents.get("text")),
+    )
+    return hashlib.sha256(repr(norm).encode("utf-8")).hexdigest()
+
+
+def build_query(req: dict) -> str:
+    parts: list[str] = [(req.get("query") or "").strip()]
+    for d in req.get("includeDomains") or []:
+        if d:
+            parts.append(f"site:{d}")
+    for d in req.get("excludeDomains") or []:
+        if d:
+            parts.append(f"-site:{d}")
+    return " ".join(p for p in parts if p)
+
+
+def _extract_highlights(body: str | None, max_n: int = 3) -> list[str]:
+    if not body:
+        return []
+    sentences = [s.strip() for s in _SENTENCE_SPLIT.split(body) if s.strip()]
+    return sentences[:max_n]
+
+
+def _favicon_for(url: str) -> str:
+    try:
+        netloc = urlparse(url).netloc
+    except ValueError:
+        netloc = ""
+    return _FAVICON.format(netloc=netloc)
+
+
+def _dgr_to_exa(r: dict, contents_highlights: bool, contents_text: bool) -> dict:
+    href = r.get("href") or ""
+    body = r.get("body") or ""
+    text = body if contents_text else ""
+    highlights = _extract_highlights(body) if contents_highlights else []
+    return {
+        "title": r.get("title") or "",
+        "url": href,
+        "id": href,
+        "text": text,
+        "highlights": highlights,
+        "highlightScores": [0.5] * len(highlights),
+        "publishedDate": None,
+        "author": None,
+        "image": None,
+        "favicon": _favicon_for(href),
+        "extras": {"links": []},
+    }
+
+
+def search(req: dict) -> dict:
+    ignored = []
+    for field in ("startPublishedDate", "endPublishedDate", "additionalQueries",
+                  "systemPrompt", "outputSchema", "stream"):
+        if req.get(field):
+            ignored.append(field)
+    if (req.get("contents") or {}).get("summary"):
+        ignored.append("contents.summary")
+    if ignored:
+        log.warning("exa_compat: ignoring unsupported fields: %s", ignored)
+
+    contents = req.get("contents") or {}
+    contents_highlights = bool(contents.get("highlights"))
+    contents_text = bool(contents.get("text"))
+
+    num_results = max(_NUM_CLAMP[0], min(_NUM_CLAMP[1], int(req.get("numResults") or 10)))
+    query = build_query(req)
+    search_type = req.get("type") or "auto"
+
+    region = "wt-wt" if search_type == "instant" else None
+    timelimit = "d" if req.get("category") == "news" else None
+
+    backends_to_try = ["duckduckgo", "auto"]
+    raw: list[dict[str, Any]] = []
+    last_err: Exception | None = None
+    for backend in backends_to_try:
+        try:
+            kwargs: dict[str, Any] = dict(
+                query=query,
+                max_results=num_results,
+                backend=backend,
+                safesearch="moderate",
+            )
+            if region:
+                kwargs["region"] = region
+            if timelimit:
+                kwargs["timelimit"] = timelimit
+            raw = list(DDGS().text(**kwargs))
+            if raw:
+                break
+        except Exception as e:
+            last_err = e
+            log.warning("exa_compat: backend %s failed: %s", backend, e)
+            continue
+
+    if not raw and last_err is not None:
+        log.error("exa_compat: all backends failed, last error: %s", last_err)
+
+    results = [_dgr_to_exa(r, contents_highlights, contents_text) for r in raw]
+    return {
+        "requestId": str(uuid.uuid4()),
+        "searchType": search_type,
+        "results": results,
+        "costDollars": {"total": 0.0},
+    }

oxe/mcp_server.py

@@ -0,0 +1,84 @@
+from typing import Any
+
+from mcp.server.mcpserver import MCPServer
+
+from . import exa_compat
+from .cache import TTLCache
+
+_cache: TTLCache | None = None
+
+
+def set_cache(c: TTLCache) -> None:
+    global _cache
+    _cache = c
+
+
+mcp = MCPServer(
+    name="oxe",
+    instructions=(
+        "Local Exa-compatible web search backed by DuckDuckGo with a TTL cache. "
+        "Query returns Exa-shaped JSON: {requestId, searchType, results, costDollars} "
+        "where each result has {title, url, id, text, highlights, highlightScores, "
+        "publishedDate, author, image, favicon, extras}. Unimplemented Exa fields "
+        "(deep search variants, contents.summary, additionalQueries, systemPrompt, "
+        "outputSchema, stream) are silently ignored."
+    ),
+)
+
+
+@mcp.tool(name="exa_search", description=(
+    "Search the web via DuckDuckGo and return Exa-shaped JSON. "
+    "Args: query (required), num_results (1-30, default 10), type ('auto'|'instant'; "
+    "deep variants ignored), contents_highlights, contents_text, include_domains, "
+    "exclude_domains, category ('news' for last 24h, else ''). "
+    "Returns {requestId, searchType, results, costDollars, _source}."
+))
+def exa_search(
+    query: str,
+    num_results: int = 10,
+    type: str = "auto",
+    contents_highlights: bool = True,
+    contents_text: bool = True,
+    include_domains: list[str] | None = None,
+    exclude_domains: list[str] | None = None,
+    category: str = "",
+) -> dict[str, Any]:
+    req: dict[str, Any] = {
+        "query": query,
+        "numResults": num_results,
+        "type": type,
+        "contents": {"highlights": contents_highlights, "text": contents_text},
+        "category": category,
+    }
+    if include_domains:
+        req["includeDomains"] = include_domains
+    if exclude_domains:
+        req["excludeDomains"] = exclude_domains
+    if _cache is None:
+        return exa_compat.search(req)
+    from .search import do_search
+    out = do_search(_cache, req)
+    return out
+
+
+@mcp.tool(name="exa_user_history", description=(
+    "Recent URLs the user has clicked from the search UI for a given query. "
+    "Use this to avoid re-researching what the user has already explored. "
+    "Args: query (optional substring match against query text), query_hash "
+    "(optional exact match), limit (1-200, default 20), since_hours (default 168 = 1 week). "
+    "Returns {clicks: [{query_hash, query, result_id, url, title, clicked_at, source}], count}."
+))
+def exa_user_history(
+    query: str = "",
+    query_hash: str = "",
+    limit: int = 20,
+    since_hours: int = 168,
+) -> dict[str, Any]:
+    if _cache is None:
+        return {"clicks": [], "count": 0, "error": "cache not initialized"}
+    qh = query_hash.strip() or None
+    qt = query.strip() or None
+    since = max(1, min(since_hours, 24 * 365))
+    lim = max(1, min(limit, 200))
+    rows = _cache.get_clicks(query_hash=qh, query_text=qt, limit=lim, since_hours=since)
+    return {"clicks": rows, "count": len(rows)}

oxe/search.py

@@ -0,0 +1,33 @@
+import logging
+import os
+from typing import Optional
+
+from . import exa_compat
+from .cache import TTLCache
+
+log = logging.getLogger(__name__)
+
+TTL_DEFAULT = int(os.getenv("OXE_TTL_DEFAULT", "3600"))
+TTL_MAX = int(os.getenv("OXE_TTL_MAX", "86400"))
+NEGATIVE_TTL = int(os.getenv("OXE_NEGATIVE_TTL", "300"))
+
+
+def do_search(cache: TTLCache, req_dict: dict, ttl: Optional[int] = None) -> dict:
+    key = exa_compat.cache_key(req_dict)
+    cached = cache.get(key)
+    if cached is not None:
+        out = dict(cached)
+        out["_source"] = "cache"
+        out["_q_hash"] = key
+        return out
+
+    response = exa_compat.search(req_dict)
+    effective_ttl = ttl if ttl is not None else (NEGATIVE_TTL if not response["results"] else TTL_DEFAULT)
+    effective_ttl = min(effective_ttl, TTL_MAX)
+    to_store = dict(response)
+    to_store["_q"] = (req_dict.get("query") or "")[:200]
+    cache.set(key, to_store, effective_ttl)
+    out = dict(response)
+    out["_source"] = "network"
+    out["_q_hash"] = key
+    return out