openscour 0.3.1__py3-none-any.whl

openscour/__init__.py

@@ -0,0 +1,38 @@
+"""OpenScour: self-hosted web search for LLMs and agents.
+
+    from openscour import OpenScour
+
+    client = OpenScour(api_key="YOUR_API_KEY")
+    results = client.search("Latest AWS Bedrock updates", max_results=10)
+"""
+
+from openscour.client import AsyncOpenScour, OpenScour
+from openscour.errors import (
+    APIConnectionError,
+    APIStatusError,
+    AuthenticationError,
+    InvalidRequestError,
+    NoResultsError,
+    OpenScourError,
+    RateLimitError,
+)
+from openscour.types import ExtractResponse, ExtractResult, FailedResult, SearchResponse, SearchResult
+
+__version__ = "0.3.1"
+
+__all__ = [
+    "OpenScour",
+    "AsyncOpenScour",
+    "SearchResponse",
+    "SearchResult",
+    "ExtractResponse",
+    "ExtractResult",
+    "FailedResult",
+    "OpenScourError",
+    "APIConnectionError",
+    "APIStatusError",
+    "AuthenticationError",
+    "InvalidRequestError",
+    "NoResultsError",
+    "RateLimitError",
+]

openscour/client.py

@@ -0,0 +1,267 @@
+from __future__ import annotations
+
+import asyncio
+import os
+import random
+import time
+from typing import Any, Literal
+
+import httpx
+
+from openscour.errors import (
+    APIConnectionError,
+    APIStatusError,
+    AuthenticationError,
+    InvalidRequestError,
+    NoResultsError,
+    RateLimitError,
+)
+from openscour.types import ExtractResponse, SearchResponse
+
+DEFAULT_BASE_URL = "http://localhost:8000"
+DEFAULT_TIMEOUT = 60.0
+DEFAULT_MAX_RETRIES = 2
+# Longest wait between retries; a longer Retry-After is raised to the caller instead.
+MAX_RETRY_WAIT = 30.0
+
+# Connection failures that happen before the server could act on the request.
+# Read timeouts are not retried: the search may still be running.
+_RETRYABLE_ERRORS = (httpx.ConnectError, httpx.ConnectTimeout, httpx.RemoteProtocolError)
+
+# Module-level so tests can skip the waiting.
+_sleep = time.sleep
+_async_sleep = asyncio.sleep
+
+SearchDepth = Literal["basic", "advanced"]
+Topic = Literal["general", "news"]
+TimeRange = Literal["day", "week", "month", "year"]
+
+
+def _search_body(
+    query: str,
+    search_depth: SearchDepth,
+    topic: Topic,
+    max_results: int,
+    include_answer: bool,
+    include_raw_content: bool,
+    include_domains: list[str] | None,
+    exclude_domains: list[str] | None,
+    time_range: TimeRange | None,
+) -> dict[str, Any]:
+    return {
+        "query": query,
+        "search_depth": search_depth,
+        "topic": topic,
+        "max_results": max_results,
+        "include_answer": include_answer,
+        "include_raw_content": include_raw_content,
+        "include_domains": include_domains or [],
+        "exclude_domains": exclude_domains or [],
+        "time_range": time_range,
+    }
+
+
+def _extract_body(urls: str | list[str], query: str | None, chunks_per_source: int) -> dict[str, Any]:
+    body: dict[str, Any] = {"urls": [urls] if isinstance(urls, str) else urls}
+    if query:
+        body["query"] = query
+        body["chunks_per_source"] = chunks_per_source
+    return body
+
+
+def _is_json(resp: httpx.Response) -> bool:
+    return resp.headers.get("content-type", "").startswith("application/json")
+
+
+def _retry_after(resp: httpx.Response) -> float | None:
+    try:
+        return max(0.0, float(resp.headers["retry-after"]))
+    except (KeyError, ValueError):
+        return None
+
+
+def _retry_wait(attempt: int, resp: httpx.Response | None = None) -> float | None:
+    """Seconds to wait before retrying, or None if this outcome should not be retried."""
+    if resp is not None:
+        # OpenScour's own 502 means no search results, which a retry won't change;
+        # a 502 page from a proxy in front of it is transient.
+        transient = resp.status_code in (429, 503, 504) or (resp.status_code == 502 and not _is_json(resp))
+        if not transient:
+            return None
+        if (wait := _retry_after(resp)) is not None:
+            return wait if wait <= MAX_RETRY_WAIT else None
+    # Exponential backoff with jitter, so many clients don't retry in step.
+    return min(MAX_RETRY_WAIT, 0.5 * 2**attempt) * random.uniform(0.5, 1.0)
+
+
+def _raise_for_status(resp: httpx.Response) -> None:
+    if resp.is_success:
+        return
+    try:
+        body = resp.json()
+        detail = body.get("detail", body) if isinstance(body, dict) else body
+    except ValueError:
+        body = detail = resp.text
+    message = detail if isinstance(detail, str) else str(detail)
+    if resp.status_code == 429:
+        raise RateLimitError(message, 429, body, retry_after=_retry_after(resp))
+    if resp.status_code == 502 and not _is_json(resp):
+        raise APIStatusError(message, 502, body)
+    error = {401: AuthenticationError, 422: InvalidRequestError, 502: NoResultsError}.get(
+        resp.status_code, APIStatusError
+    )
+    raise error(message, resp.status_code, body)
+
+
+class _BaseClient:
+    def __init__(self, api_key: str | None, base_url: str | None, max_retries: int):
+        self.max_retries = max_retries
+        self.api_key = api_key if api_key is not None else os.environ.get("OPENSCOUR_API_KEY")
+        self.base_url = (base_url or os.environ.get("OPENSCOUR_BASE_URL") or DEFAULT_BASE_URL).rstrip("/")
+
+    def _headers(self) -> dict[str, str]:
+        return {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}
+
+
+class OpenScour(_BaseClient):
+    """Client for an OpenScour server.
+
+    >>> client = OpenScour(api_key="...", base_url="http://localhost:8000")
+    >>> response = client.search("Latest AWS Bedrock updates", max_results=10)
+    >>> response.results[0].content
+
+    ``api_key`` and ``base_url`` default to the ``OPENSCOUR_API_KEY`` and
+    ``OPENSCOUR_BASE_URL`` environment variables. Requests that fail to connect
+    or get 429, 503 or 504 are retried up to ``max_retries`` times, honoring the
+    server's Retry-After.
+    """
+
+    def __init__(
+        self,
+        api_key: str | None = None,
+        base_url: str | None = None,
+        timeout: float = DEFAULT_TIMEOUT,
+        http_client: httpx.Client | None = None,
+        max_retries: int = DEFAULT_MAX_RETRIES,
+    ):
+        super().__init__(api_key, base_url, max_retries)
+        self._client = http_client or httpx.Client(timeout=timeout)
+
+    def _post(self, path: str, body: dict[str, Any]) -> dict[str, Any]:
+        attempt = 0
+        while True:
+            try:
+                resp = self._client.post(f"{self.base_url}{path}", json=body, headers=self._headers())
+            except httpx.HTTPError as e:
+                if isinstance(e, _RETRYABLE_ERRORS) and attempt < self.max_retries:
+                    _sleep(_retry_wait(attempt))
+                    attempt += 1
+                    continue
+                raise APIConnectionError(f"Could not reach OpenScour at {self.base_url}: {e}") from e
+            if attempt < self.max_retries and (wait := _retry_wait(attempt, resp)) is not None:
+                _sleep(wait)
+                attempt += 1
+                continue
+            _raise_for_status(resp)
+            return resp.json()
+
+    def search(
+        self,
+        query: str,
+        *,
+        search_depth: SearchDepth = "basic",
+        topic: Topic = "general",
+        max_results: int = 5,
+        include_answer: bool = False,
+        include_raw_content: bool = False,
+        include_domains: list[str] | None = None,
+        exclude_domains: list[str] | None = None,
+        time_range: TimeRange | None = None,
+    ) -> SearchResponse:
+        body = _search_body(
+            query, search_depth, topic, max_results, include_answer,
+            include_raw_content, include_domains, exclude_domains, time_range,
+        )  # fmt: skip
+        return SearchResponse.from_dict(self._post("/search", body))
+
+    def extract(
+        self, urls: str | list[str], *, query: str | None = None, chunks_per_source: int = 3
+    ) -> ExtractResponse:
+        """Clean text of each page; with ``query``, only its most relevant passages."""
+        return ExtractResponse.from_dict(self._post("/extract", _extract_body(urls, query, chunks_per_source)))
+
+    def close(self) -> None:
+        self._client.close()
+
+    def __enter__(self) -> OpenScour:
+        return self
+
+    def __exit__(self, *exc: object) -> None:
+        self.close()
+
+
+class AsyncOpenScour(_BaseClient):
+    """Async version of :class:`OpenScour`; same methods, awaited."""
+
+    def __init__(
+        self,
+        api_key: str | None = None,
+        base_url: str | None = None,
+        timeout: float = DEFAULT_TIMEOUT,
+        http_client: httpx.AsyncClient | None = None,
+        max_retries: int = DEFAULT_MAX_RETRIES,
+    ):
+        super().__init__(api_key, base_url, max_retries)
+        self._client = http_client or httpx.AsyncClient(timeout=timeout)
+
+    async def _post(self, path: str, body: dict[str, Any]) -> dict[str, Any]:
+        attempt = 0
+        while True:
+            try:
+                resp = await self._client.post(f"{self.base_url}{path}", json=body, headers=self._headers())
+            except httpx.HTTPError as e:
+                if isinstance(e, _RETRYABLE_ERRORS) and attempt < self.max_retries:
+                    await _async_sleep(_retry_wait(attempt))
+                    attempt += 1
+                    continue
+                raise APIConnectionError(f"Could not reach OpenScour at {self.base_url}: {e}") from e
+            if attempt < self.max_retries and (wait := _retry_wait(attempt, resp)) is not None:
+                await _async_sleep(wait)
+                attempt += 1
+                continue
+            _raise_for_status(resp)
+            return resp.json()
+
+    async def search(
+        self,
+        query: str,
+        *,
+        search_depth: SearchDepth = "basic",
+        topic: Topic = "general",
+        max_results: int = 5,
+        include_answer: bool = False,
+        include_raw_content: bool = False,
+        include_domains: list[str] | None = None,
+        exclude_domains: list[str] | None = None,
+        time_range: TimeRange | None = None,
+    ) -> SearchResponse:
+        body = _search_body(
+            query, search_depth, topic, max_results, include_answer,
+            include_raw_content, include_domains, exclude_domains, time_range,
+        )  # fmt: skip
+        return SearchResponse.from_dict(await self._post("/search", body))
+
+    async def extract(
+        self, urls: str | list[str], *, query: str | None = None, chunks_per_source: int = 3
+    ) -> ExtractResponse:
+        """Clean text of each page; with ``query``, only its most relevant passages."""
+        return ExtractResponse.from_dict(await self._post("/extract", _extract_body(urls, query, chunks_per_source)))
+
+    async def close(self) -> None:
+        await self._client.aclose()
+
+    async def __aenter__(self) -> AsyncOpenScour:
+        return self
+
+    async def __aexit__(self, *exc: object) -> None:
+        await self.close()

openscour/errors.py

@@ -0,0 +1,35 @@
+class OpenScourError(Exception):
+    """Base class for all errors raised by the OpenScour client."""
+
+
+class APIConnectionError(OpenScourError):
+    """The OpenScour server could not be reached."""
+
+
+class APIStatusError(OpenScourError):
+    """The server returned an error response."""
+
+    def __init__(self, message: str, status_code: int, body: object = None):
+        super().__init__(f"{status_code}: {message}")
+        self.status_code = status_code
+        self.body = body
+
+
+class AuthenticationError(APIStatusError):
+    """The API key is missing or invalid (HTTP 401)."""
+
+
+class InvalidRequestError(APIStatusError):
+    """The request was rejected, e.g. an empty query (HTTP 422)."""
+
+
+class NoResultsError(APIStatusError):
+    """No search backend returned results (HTTP 502)."""
+
+
+class RateLimitError(APIStatusError):
+    """Too many requests (HTTP 429). ``retry_after`` is the seconds the server asked to wait, if it said."""
+
+    def __init__(self, message: str, status_code: int, body: object = None, retry_after: float | None = None):
+        super().__init__(message, status_code, body)
+        self.retry_after = retry_after

openscour/mcp_server.py

@@ -0,0 +1,161 @@
+"""MCP server that gives AI agents OpenScour's search and extract as tools.
+
+Runs over stdio and talks to an OpenScour server through the Python client,
+configured with OPENSCOUR_BASE_URL and OPENSCOUR_API_KEY.
+
+    claude mcp add openscour -- openscour-mcp
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from typing import Literal
+
+from mcp.server.mcpserver import MCPServer
+from mcp.server.mcpserver.exceptions import ToolError
+from mcp.types import ToolAnnotations
+
+from openscour.client import AsyncOpenScour
+from openscour.errors import APIConnectionError, OpenScourError
+from openscour.types import ExtractResponse, SearchResponse
+
+INSTRUCTIONS = (
+    "OpenScour searches the web and returns ranked passages from real pages. "
+    "Use `search` for current events, recent releases, facts you are unsure of, "
+    "or anything after your training data. Use `extract` to read the full text "
+    "of specific URLs. Cite the URLs of the passages you rely on."
+)
+
+# Upper bound on extracted page text per URL, so one long page can't flood the context.
+MAX_EXTRACT_CHARS = 20_000
+
+server = MCPServer(name="openscour", instructions=INSTRUCTIONS)
+
+_READ_ONLY_WEB = ToolAnnotations(readOnlyHint=True, openWorldHint=True)
+
+
+# One client for the session, so tool calls reuse open connections. Tied to the
+# event loop it was made on, since an async HTTP client can't move between loops.
+_shared: tuple[asyncio.AbstractEventLoop, AsyncOpenScour] | None = None
+
+
+def _shared_client() -> AsyncOpenScour:
+    global _shared
+    loop = asyncio.get_running_loop()
+    if _shared is None or _shared[0] is not loop:
+        _shared = (loop, AsyncOpenScour())
+    return _shared[1]
+
+
+@asynccontextmanager
+async def _client() -> AsyncIterator[AsyncOpenScour]:
+    """The shared OpenScour client, with errors that reach the agent as readable tool errors.
+
+    Other exceptions only show the agent "Error executing tool", which gives it
+    nothing to act on (e.g. that the OpenScour server isn't running).
+    """
+    try:
+        yield _shared_client()
+    except APIConnectionError as e:
+        raise ToolError(f"{e}. Is the OpenScour server running?") from e
+    except OpenScourError as e:
+        raise ToolError(str(e)) from e
+
+
+def format_search(response: SearchResponse) -> str:
+    if not response.results:
+        return f"No results for {response.query!r}."
+    parts = []
+    if response.answer:
+        parts.append(f"Answer: {response.answer}")
+    for i, r in enumerate(response.results, start=1):
+        lines = [f"[{i}] {r.title}", f"URL: {r.url}"]
+        if r.published_date:
+            lines.append(f"Published: {r.published_date}")
+        lines.append(r.content)
+        if r.raw_content:
+            lines.append(f"Full text:\n{r.raw_content[:MAX_EXTRACT_CHARS]}")
+        parts.append("\n".join(lines))
+    return "\n\n".join(parts)
+
+
+def format_extract(response: ExtractResponse) -> str:
+    parts = []
+    for r in response.results:
+        text = r.raw_content
+        if len(text) > MAX_EXTRACT_CHARS:
+            text = text[:MAX_EXTRACT_CHARS] + f"\n[... truncated, {len(r.raw_content)} characters in total]"
+        parts.append(f"URL: {r.url}\n{text}")
+    for f in response.failed_results:
+        parts.append(f"URL: {f.url}\nCould not extract: {f.error}")
+    return "\n\n---\n\n".join(parts) or "No URLs were extracted."
+
+
+@server.tool(
+    description=(
+        "Search the web and get the most relevant passage from each result page. "
+        "Use for current or factual information. Results are numbered with title, URL and passage."
+    ),
+    annotations=_READ_ONLY_WEB,
+)
+async def search(
+    query: str,
+    max_results: int = 5,
+    search_depth: Literal["basic", "advanced"] = "basic",
+    topic: Literal["general", "news"] = "general",
+    time_range: Literal["day", "week", "month", "year"] | None = None,
+    include_domains: list[str] | None = None,
+    exclude_domains: list[str] | None = None,
+) -> str:
+    """
+    Args:
+        query: What to search for, e.g. "Latest AWS Bedrock updates".
+        max_results: Number of results, 1 to 20.
+        search_depth: "advanced" reads more pages and returns more passages per result; slower.
+        topic: "news" for recent news coverage.
+        time_range: Only results from the last day, week, month or year.
+        include_domains: Only return results from these domains.
+        exclude_domains: Never return results from these domains.
+    """
+    async with _client() as client:
+        response = await client.search(
+            query,
+            max_results=max_results,
+            search_depth=search_depth,
+            topic=topic,
+            time_range=time_range,
+            include_domains=include_domains,
+            exclude_domains=exclude_domains,
+        )
+    return format_search(response)
+
+
+@server.tool(
+    description=(
+        "Read web pages. Pass `query` with what you are looking for to get only the most "
+        "relevant passages of each page (short, recommended); without it you get the full "
+        "page text, which can be very long."
+    ),
+    annotations=_READ_ONLY_WEB,
+)
+async def extract(urls: list[str], query: str | None = None, chunks_per_source: int = 3) -> str:
+    """
+    Args:
+        urls: Up to 20 http(s) URLs to read.
+        query: What you need from the pages, e.g. "new Bedrock models in October 2026".
+        chunks_per_source: With a query, how many passages to return per page (1 to 10).
+    """
+    async with _client() as client:
+        response = await client.extract(urls, query=query, chunks_per_source=chunks_per_source)
+    return format_extract(response)
+
+
+def main() -> None:
+    """Entry point for the ``openscour-mcp`` command."""
+    server.run("stdio")
+
+
+if __name__ == "__main__":
+    main()

openscour/server/answer.py

@@ -0,0 +1,56 @@
+"""Optional answer synthesis with Claude (Amazon Bedrock or the Claude API)."""
+
+import logging
+from functools import lru_cache
+
+import anthropic
+
+from openscour.server.config import settings
+from openscour.server.ranker import Ranked
+
+log = logging.getLogger(__name__)
+
+SYSTEM = (
+    "You answer a search query using only the numbered sources provided. "
+    "Write a direct, factual answer in 1-4 sentences. Cite sources inline as [1], [2]. "
+    "If the sources do not contain the answer, say so briefly."
+)
+
+
+@lru_cache(maxsize=1)
+def _client():
+    if settings.answer_provider == "bedrock":
+        return anthropic.AsyncAnthropicBedrockMantle(aws_region=settings.aws_region)
+    return anthropic.AsyncAnthropic()
+
+
+def _model() -> str:
+    m = settings.answer_model
+    if settings.answer_provider == "bedrock" and not m.startswith("anthropic."):
+        return f"anthropic.{m}"
+    return m
+
+
+async def generate_answer(query: str, results: list[Ranked]) -> str | None:
+    if settings.answer_provider == "none" or not results:
+        return None
+
+    sources = "\n\n".join(
+        f"[{i}] {r.doc.title}\n{r.doc.url}\n{r.content}" for i, r in enumerate(results, start=1)
+    )
+    messages = [{"role": "user", "content": f"Sources:\n\n{sources}\n\nQuery: {query}"}]
+    try:
+        response = await _client().messages.create(
+            model=_model(),
+            max_tokens=8000,
+            system=SYSTEM,
+            output_config={"effort": "low"},
+            messages=messages,
+        )
+    except anthropic.APIError as e:
+        log.warning("answer generation failed: %s", e)
+        return None
+
+    if response.stop_reason == "refusal":
+        return None
+    return "".join(b.text for b in response.content if b.type == "text").strip() or None

openscour/server/cache.py

@@ -0,0 +1,33 @@
+import time
+from collections import OrderedDict
+from typing import Any
+
+
+class TTLCache:
+    """Small in-process LRU cache with per-entry expiry.
+
+    Good enough for a single container. Swap for Redis (ElastiCache) once
+    you run more than one API task.
+    """
+
+    def __init__(self, ttl_seconds: int, max_items: int = 2048):
+        self.ttl = ttl_seconds
+        self.max_items = max_items
+        self._data: OrderedDict[str, tuple[float, Any]] = OrderedDict()
+
+    def get(self, key: str) -> Any | None:
+        item = self._data.get(key)
+        if item is None:
+            return None
+        expires_at, value = item
+        if expires_at < time.monotonic():
+            del self._data[key]
+            return None
+        self._data.move_to_end(key)
+        return value
+
+    def set(self, key: str, value: Any) -> None:
+        self._data[key] = (time.monotonic() + self.ttl, value)
+        self._data.move_to_end(key)
+        while len(self._data) > self.max_items:
+            self._data.popitem(last=False)

openscour/server/config.py

@@ -0,0 +1,59 @@
+from typing import Literal
+
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+from openscour import __version__
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", extra="ignore")
+
+    # Comma-separated list of keys clients must send as "Authorization: Bearer <key>".
+    # Leave empty to disable auth (local development).
+    api_keys: str = ""
+
+    searxng_url: str = "http://localhost:8888"
+    # Used only when SearXNG errors out or returns nothing.
+    duckduckgo_fallback: bool = True
+
+    # include_answer backend. "none" makes include_answer return null.
+    answer_provider: Literal["none", "bedrock", "anthropic"] = "none"
+    answer_model: str = "claude-opus-5-5"
+    aws_region: str = "us-east-1"
+
+    fetch_timeout: float = 8.0
+    fetch_concurrency: int = 10
+    extract_workers: int = 4
+    # Basic searches answer after this many seconds and use search snippets for
+    # pages that aren't ready; those pages keep loading in the background for the cache.
+    basic_fetch_deadline: float = 2.0
+    max_page_bytes: int = 3_000_000
+    cache_ttl_seconds: int = 900
+    # Pages being fetched at once across all requests; more wait their turn.
+    # Requests beyond max_inflight_fetches use search snippets instead.
+    max_inflight_fetches: int = 200
+    # Extraction workers are replaced after this many pages, so memory
+    # fragmentation in lxml can't build up in a long-running server.
+    extract_max_tasks_per_child: int = 500
+
+    # Identify honestly to the sites OpenScour reads, and follow their robots.txt.
+    user_agent: str = f"OpenScour/{__version__} (+https://github.com/ivipinmishra/openscour)"
+    respect_robots_txt: bool = True
+    # Pages are only fetched from public internet addresses (SSRF protection).
+    # Set to true only for a trusted, internal-only deployment that must read
+    # intranet pages; never on a server untrusted callers can reach.
+    allow_private_network_fetches: bool = False
+
+    # Requests per minute for each API key (or each client IP when auth is off),
+    # across /search and /extract. 0 turns rate limiting off.
+    rate_limit_per_minute: int = 60
+    # Take the client IP from the X-Forwarded-For header set by a load balancer.
+    # Only enable behind a proxy that sets it: clients can forge it otherwise.
+    trust_forwarded_for: bool = False
+
+    @property
+    def api_key_set(self) -> set[str]:
+        return {k.strip() for k in self.api_keys.split(",") if k.strip()}
+
+
+settings = Settings()