blopus 0.1.0__py3-none-any.whl

blopus/__init__.py

@@ -0,0 +1,61 @@
+"""Blopus — official Python SDK for the Blopus web search + fetch API.
+
+Quickstart::
+
+    from blopus import Blopus
+
+    client = Blopus(api_key="blp_live_...")   # or set BLOPUS_API_KEY
+    for hit in client.search("latest ai news", freshness="pd"):
+        print(hit.title, hit.url)
+
+    doc = client.fetch("https://example.com/article")
+
+The SDK calls only two data-plane endpoints — ``POST /v1/search`` and
+``POST /v1/fetch`` on https://api.blopus.ai.
+"""
+from __future__ import annotations
+
+from ._async import AsyncBlopus
+from ._version import __version__
+from .client import Blopus
+from .exceptions import (
+    APIConnectionError,
+    AuthError,
+    BadRequestError,
+    BlopusError,
+    NotFoundError,
+    QuotaError,
+    RateLimitError,
+    ServerError,
+)
+from .mcp import mcp_config, print_mcp_config
+from .models import (
+    BatchFetchResponse,
+    FetchFailure,
+    FetchResult,
+    SearchResponse,
+    SearchResult,
+)
+
+__all__ = [
+    "__version__",
+    "Blopus",
+    "AsyncBlopus",
+    "mcp_config",
+    "print_mcp_config",
+    # models
+    "SearchResponse",
+    "SearchResult",
+    "FetchResult",
+    "FetchFailure",
+    "BatchFetchResponse",
+    # exceptions
+    "BlopusError",
+    "APIConnectionError",
+    "AuthError",
+    "BadRequestError",
+    "NotFoundError",
+    "RateLimitError",
+    "QuotaError",
+    "ServerError",
+]

blopus/_async.py

@@ -0,0 +1,139 @@
+"""Asynchronous Blopus client (httpx.AsyncClient). Mirrors :class:`Blopus` exactly.
+
+    import asyncio
+    from blopus import AsyncBlopus
+
+    async def main():
+        async with AsyncBlopus() as client:
+            res = await client.search("openai news", freshness="pw")
+            docs = await client.fetch([r.url for r in res])
+
+    asyncio.run(main())
+"""
+from __future__ import annotations
+
+import asyncio
+from typing import Optional, Sequence, Union
+
+import httpx
+
+from . import _common as C
+from .exceptions import APIConnectionError
+from .models import BatchFetchResponse, FetchFailure, FetchResult, SearchResponse
+
+
+class AsyncBlopus:
+    """Async client for the Blopus search + fetch API. Same surface as :class:`Blopus`."""
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        *,
+        base_url: Optional[str] = None,
+        timeout: float = C.DEFAULT_TIMEOUT,
+        max_retries: int = C.DEFAULT_MAX_RETRIES,
+        chunk_delay: float = C.DEFAULT_CHUNK_DELAY,
+        user_agent: str = C.USER_AGENT,
+    ) -> None:
+        self._api_key = C.resolve_api_key(api_key)
+        self.base_url = C.resolve_base_url(base_url)
+        self.max_retries = max_retries
+        self.chunk_delay = chunk_delay
+        self._http = httpx.AsyncClient(
+            base_url=self.base_url,
+            timeout=timeout,
+            headers=C.build_headers(self._api_key, user_agent),
+        )
+
+    async def aclose(self) -> None:
+        await self._http.aclose()
+
+    async def __aenter__(self) -> "AsyncBlopus":
+        return self
+
+    async def __aexit__(self, *exc) -> None:
+        await self.aclose()
+
+    async def _post(self, path: str, body: dict) -> dict:
+        attempt = 0
+        while True:
+            status: Optional[int] = None
+            retry_after: Optional[int] = None
+            try:
+                resp = await self._http.post(path, json=body)
+                status = resp.status_code
+                retry_after = C.parse_retry_after(resp.headers.get("Retry-After"))
+                if 200 <= status < 300:
+                    return C.parse_json_or_raise(status, resp.text, retry_after)
+            except httpx.HTTPError as exc:
+                if not C.should_retry(None, attempt, self.max_retries):
+                    raise APIConnectionError(f"Request failed: {exc}") from exc
+                await asyncio.sleep(C.backoff_seconds(attempt, None))
+                attempt += 1
+                continue
+
+            if C.should_retry(status, attempt, self.max_retries):
+                await asyncio.sleep(C.backoff_seconds(attempt, retry_after))
+                attempt += 1
+                continue
+            return C.parse_json_or_raise(status, resp.text, retry_after)
+
+    async def search(
+        self,
+        query: str,
+        *,
+        count: int = 10,
+        freshness: str = "all",
+        include_domains: Optional[Sequence[str]] = None,
+        exclude_domains: Optional[Sequence[str]] = None,
+        start_date: Optional[str] = None,
+        end_date: Optional[str] = None,
+        language: Optional[str] = None,
+        offset: int = 0,
+        include_excerpt: bool = False,
+        excerpt_chars: Optional[int] = None,
+    ) -> SearchResponse:
+        body = C.build_search_body(
+            query,
+            count=count,
+            freshness=freshness,
+            include_domains=include_domains,
+            exclude_domains=exclude_domains,
+            start_date=start_date,
+            end_date=end_date,
+            language=language,
+            offset=offset,
+            include_excerpt=include_excerpt,
+            excerpt_chars=excerpt_chars,
+        )
+        return SearchResponse.from_dict(await self._post("/v1/search", body))
+
+    async def fetch(
+        self, url_or_urls: Union[str, Sequence[str]]
+    ) -> Union[FetchResult, BatchFetchResponse]:
+        is_batch, urls, single = C.normalize_fetch_urls(url_or_urls)
+        if not is_batch:
+            data = await self._post("/v1/fetch", {"url": single})
+            return FetchResult.from_dict(data)
+        return await self._fetch_batch(urls)
+
+    async def _fetch_batch(self, urls: Sequence[str]) -> BatchFetchResponse:
+        chunks = C.chunk_urls(urls)
+        merged_results: list[FetchResult] = []
+        merged_failed: list[FetchFailure] = []
+        remaining: Optional[int] = None
+        for i, chunk in enumerate(chunks):
+            if i > 0 and self.chunk_delay > 0:
+                await asyncio.sleep(self.chunk_delay)
+            data = await self._post("/v1/fetch", {"urls": list(chunk)})
+            part = BatchFetchResponse.from_dict(data)
+            merged_results.extend(part.results)
+            merged_failed.extend(part.failed_results)
+            if part.remaining_quota is not None:
+                remaining = part.remaining_quota
+        return BatchFetchResponse(
+            results=merged_results,
+            failed_results=merged_failed,
+            count=len(merged_results),
+            remaining_quota=remaining,
+        )

blopus/_common.py

@@ -0,0 +1,155 @@
+"""Shared, transport-agnostic helpers used by both the sync and async clients.
+
+Keeping request-building, response-parsing, retry policy and error mapping here
+guarantees the two clients behave identically — the only difference between them
+is ``httpx.Client`` vs ``httpx.AsyncClient``.
+"""
+from __future__ import annotations
+
+import os
+import random
+from typing import Any, Dict, List, Optional, Sequence, Union
+
+from ._version import __version__
+from .exceptions import BlopusError, error_from_response
+
+DEFAULT_BASE_URL = "https://api.blopus.ai"
+DEFAULT_MCP_URL = "https://mcp.blopus.ai"
+DEFAULT_TIMEOUT = 30.0
+DEFAULT_MAX_RETRIES = 2
+ENV_API_KEY = "BLOPUS_API_KEY"
+ENV_BASE_URL = "BLOPUS_BASE_URL"
+
+# Server-enforced cap: at most 50 URLs per /v1/fetch call.
+BATCH_URL_LIMIT = 50
+# Small courtesy delay between auto-chunked batches to respect per-second limits.
+DEFAULT_CHUNK_DELAY = 0.25
+
+# A *real* User-Agent is mandatory: Cloudflare 1010-blocks default library UAs
+# (python-httpx/..., node-fetch, etc.) on api.blopus.ai.
+USER_AGENT = f"blopus-python/{__version__}"
+
+# Retry these HTTP statuses with backoff (transient / rate-limited).
+RETRY_STATUSES = frozenset({429, 500, 502, 503, 504})
+
+
+def resolve_api_key(api_key: Optional[str]) -> str:
+    key = api_key or os.environ.get(ENV_API_KEY)
+    if not key:
+        raise BlopusError(
+            "No API key provided. Pass api_key=... or set the "
+            f"{ENV_API_KEY} environment variable.",
+            code="no_api_key",
+        )
+    return key
+
+
+def resolve_base_url(base_url: Optional[str]) -> str:
+    return (base_url or os.environ.get(ENV_BASE_URL) or DEFAULT_BASE_URL).rstrip("/")
+
+
+def build_headers(api_key: str, user_agent: str = USER_AGENT) -> Dict[str, str]:
+    return {
+        "Authorization": f"Bearer {api_key}",
+        "User-Agent": user_agent,
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+    }
+
+
+def build_search_body(
+    query: str,
+    *,
+    count: int = 10,
+    freshness: str = "all",
+    include_domains: Optional[Sequence[str]] = None,
+    exclude_domains: Optional[Sequence[str]] = None,
+    start_date: Optional[str] = None,
+    end_date: Optional[str] = None,
+    language: Optional[str] = None,
+    offset: int = 0,
+    include_excerpt: bool = False,
+    excerpt_chars: Optional[int] = None,
+) -> Dict[str, Any]:
+    """Assemble the JSON body for ``POST /v1/search`` (omitting unset optionals)."""
+    body: Dict[str, Any] = {
+        "query": query,
+        "count": count,
+        "freshness": freshness,
+        "offset": offset,
+    }
+    if include_domains:
+        body["include_domains"] = list(include_domains)
+    if exclude_domains:
+        body["exclude_domains"] = list(exclude_domains)
+    if start_date is not None:
+        body["start_date"] = start_date
+    if end_date is not None:
+        body["end_date"] = end_date
+    if language is not None:
+        body["language"] = language
+    if include_excerpt:
+        body["include_excerpt"] = True
+    if excerpt_chars is not None:
+        body["excerpt_chars"] = excerpt_chars
+    return body
+
+
+def normalize_fetch_urls(
+    url_or_urls: Union[str, Sequence[str]],
+) -> tuple[bool, List[str], Optional[str]]:
+    """Return (is_batch, url_list, single_url).
+
+    A single string is a single fetch; any sequence is a batch (even length 1).
+    """
+    if isinstance(url_or_urls, str):
+        return False, [url_or_urls], url_or_urls
+    urls = [str(u) for u in url_or_urls]
+    if not urls:
+        raise BlopusError("fetch() requires at least one URL.", code="bad_request")
+    return True, urls, None
+
+
+def chunk_urls(urls: Sequence[str], size: int = BATCH_URL_LIMIT) -> List[List[str]]:
+    return [list(urls[i : i + size]) for i in range(0, len(urls), size)]
+
+
+def parse_json_or_raise(status: int, text: str, retry_after: Optional[int]) -> Dict[str, Any]:
+    """Parse a response body; raise a typed error for non-2xx statuses."""
+    import json
+
+    body: Optional[Dict[str, Any]]
+    try:
+        body = json.loads(text) if text else None
+    except (ValueError, TypeError):
+        body = None
+    if 200 <= status < 300:
+        if not isinstance(body, dict):
+            raise error_from_response(status, {"error": {"code": "invalid_response",
+                                                         "message": "Malformed JSON from API."}})
+        return body
+    raise error_from_response(status, body, retry_after=retry_after)
+
+
+def parse_retry_after(value: Optional[str]) -> Optional[int]:
+    if not value:
+        return None
+    try:
+        return max(0, int(float(value)))
+    except (TypeError, ValueError):
+        return None
+
+
+def backoff_seconds(attempt: int, retry_after: Optional[int]) -> float:
+    """Exponential backoff with jitter; honor server Retry-After when present."""
+    if retry_after is not None:
+        return float(retry_after)
+    return min(8.0, (2 ** attempt) * 0.5) + random.uniform(0, 0.25)
+
+
+def should_retry(status: Optional[int], attempt: int, max_retries: int) -> bool:
+    if attempt >= max_retries:
+        return False
+    if status is None:  # connection error
+        return True
+    return status in RETRY_STATUSES

blopus/_version.py

@@ -0,0 +1,8 @@
+"""Single source of truth for the package version.
+
+Bump this on every release (see PUBLISH.md). It also feeds the User-Agent
+header, so a correct value here is what keeps Cloudflare from 1010-blocking
+requests as an unidentified library.
+"""
+
+__version__ = "0.1.0"

blopus/cli.py

@@ -0,0 +1,136 @@
+"""Command-line interface for Blopus.
+
+    blopus search "who won the game" --count 5 --freshness pd
+    blopus search "openai" --include-domains techcrunch.com,theverge.com --json
+    blopus fetch https://example.com/a https://example.com/b
+    blopus mcp-config
+
+Reads the API key from --api-key or the BLOPUS_API_KEY environment variable.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from typing import List, Optional
+
+from ._version import __version__
+from .client import Blopus
+from .exceptions import BlopusError
+from .mcp import mcp_config
+from .models import BatchFetchResponse
+
+
+def _csv(value: Optional[str]) -> Optional[List[str]]:
+    if not value:
+        return None
+    return [v.strip() for v in value.split(",") if v.strip()]
+
+
+def _cmd_search(args: argparse.Namespace) -> int:
+    client = Blopus(api_key=args.api_key, base_url=args.base_url)
+    res = client.search(
+        args.query,
+        count=args.count,
+        freshness=args.freshness,
+        include_domains=_csv(args.include_domains),
+        exclude_domains=_csv(args.exclude_domains),
+        language=args.language,
+        offset=args.offset,
+        include_excerpt=args.include_excerpt,
+    )
+    if args.json:
+        print(json.dumps(res.raw, indent=2, ensure_ascii=False))
+    else:
+        if not res.results:
+            print("No results.")
+        for i, r in enumerate(res.results, 1):
+            print(f"{i}. {r.title}")
+            print(f"   {r.url}")
+            if r.snippet:
+                print(f"   {r.snippet}")
+            print()
+        if res.remaining_quota is not None:
+            print(f"[remaining quota: {res.remaining_quota}]", file=sys.stderr)
+    return 0
+
+
+def _cmd_fetch(args: argparse.Namespace) -> int:
+    client = Blopus(api_key=args.api_key, base_url=args.base_url)
+    result = client.fetch(args.urls if len(args.urls) > 1 else args.urls[0])
+    if args.json:
+        raw = result.raw if not isinstance(result, BatchFetchResponse) else {
+            "results": [r.raw for r in result.results],
+            "failed_results": [f.raw for f in result.failed_results],
+            "count": result.count,
+            "remaining_quota": result.remaining_quota,
+        }
+        print(json.dumps(raw, indent=2, ensure_ascii=False))
+        return 0
+    if isinstance(result, BatchFetchResponse):
+        for r in result.results:
+            print(f"# {r.title}\n{r.url}\n{r.content[:2000]}\n")
+        if result.failed_results:
+            print("Not found:", ", ".join(f.url for f in result.failed_results),
+                  file=sys.stderr)
+    else:
+        print(f"# {result.title}\n{result.url}\n\n{result.content}")
+    return 0
+
+
+def _cmd_mcp_config(args: argparse.Namespace) -> int:
+    print(json.dumps(mcp_config(args.api_key), indent=2))
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(prog="blopus", description="Blopus web search + fetch API.")
+    p.add_argument("--version", action="version", version=f"blopus {__version__}")
+
+    # Shared auth options are attached to each subcommand (via a parent parser) so
+    # they work in the natural position, e.g. `blopus search "q" --api-key ...`.
+    common = argparse.ArgumentParser(add_help=False)
+    common.add_argument("--api-key", default=None, help="API key (or set BLOPUS_API_KEY).")
+    common.add_argument("--base-url", default=None, help="Override API base URL.")
+
+    sub = p.add_subparsers(dest="command", required=True)
+
+    s = sub.add_parser("search", help="Run a web search.", parents=[common])
+    s.add_argument("query")
+    s.add_argument("--count", type=int, default=10)
+    s.add_argument("--freshness", default="all",
+                   choices=["pd", "pw", "pm", "p3m", "p1y", "all"])
+    s.add_argument("--include-domains", default=None, help="Comma-separated allowlist.")
+    s.add_argument("--exclude-domains", default=None, help="Comma-separated blocklist.")
+    s.add_argument("--language", default=None, help="Language code, e.g. en.")
+    s.add_argument("--offset", type=int, default=0)
+    s.add_argument("--include-excerpt", action="store_true", help="Request longer excerpts.")
+    s.add_argument("--json", action="store_true", help="Emit raw JSON.")
+    s.set_defaults(func=_cmd_search)
+
+    f = sub.add_parser("fetch", help="Fetch indexed content for one or more URLs.",
+                       parents=[common])
+    f.add_argument("urls", nargs="+")
+    f.add_argument("--json", action="store_true", help="Emit raw JSON.")
+    f.set_defaults(func=_cmd_fetch)
+
+    m = sub.add_parser("mcp-config", help="Print the MCP server config JSON.",
+                       parents=[common])
+    m.set_defaults(func=_cmd_mcp_config)
+    return p
+
+
+def main(argv: Optional[List[str]] = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        return args.func(args)
+    except BlopusError as exc:
+        print(f"Error: {exc}", file=sys.stderr)
+        return 1
+    except KeyboardInterrupt:  # pragma: no cover
+        return 130
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

blopus/client.py

@@ -0,0 +1,158 @@
+"""Synchronous Blopus client.
+
+    from blopus import Blopus
+
+    client = Blopus()                      # reads BLOPUS_API_KEY
+    res = client.search("who won the game last night", freshness="pd")
+    for hit in res:
+        print(hit.title, hit.url)
+
+    doc = client.fetch("https://example.com/article")
+    batch = client.fetch(["https://a.com", "https://b.com"])   # auto-chunks > 50
+"""
+from __future__ import annotations
+
+import time
+from typing import Optional, Sequence, Union
+
+import httpx
+
+from . import _common as C
+from .exceptions import APIConnectionError, BlopusError
+from .models import BatchFetchResponse, FetchFailure, FetchResult, SearchResponse
+
+
+class Blopus:
+    """Synchronous client for the Blopus search + fetch API.
+
+    Args:
+        api_key: ``blp_live_...`` key. Falls back to ``$BLOPUS_API_KEY``.
+        base_url: API origin. Defaults to ``https://api.blopus.ai``.
+        timeout: Per-request timeout in seconds.
+        max_retries: Retries on 429 / 5xx / connection errors (honors Retry-After).
+        chunk_delay: Delay between auto-chunked batch-fetch calls (seconds).
+    """
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        *,
+        base_url: Optional[str] = None,
+        timeout: float = C.DEFAULT_TIMEOUT,
+        max_retries: int = C.DEFAULT_MAX_RETRIES,
+        chunk_delay: float = C.DEFAULT_CHUNK_DELAY,
+        user_agent: str = C.USER_AGENT,
+    ) -> None:
+        self._api_key = C.resolve_api_key(api_key)
+        self.base_url = C.resolve_base_url(base_url)
+        self.max_retries = max_retries
+        self.chunk_delay = chunk_delay
+        self._http = httpx.Client(
+            base_url=self.base_url,
+            timeout=timeout,
+            headers=C.build_headers(self._api_key, user_agent),
+        )
+
+    # -- lifecycle ---------------------------------------------------------- #
+    def close(self) -> None:
+        self._http.close()
+
+    def __enter__(self) -> "Blopus":
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self.close()
+
+    # -- transport ---------------------------------------------------------- #
+    def _post(self, path: str, body: dict) -> dict:
+        attempt = 0
+        while True:
+            status: Optional[int] = None
+            retry_after: Optional[int] = None
+            try:
+                resp = self._http.post(path, json=body)
+                status = resp.status_code
+                retry_after = C.parse_retry_after(resp.headers.get("Retry-After"))
+                if 200 <= status < 300:
+                    return C.parse_json_or_raise(status, resp.text, retry_after)
+            except httpx.HTTPError as exc:
+                if not C.should_retry(None, attempt, self.max_retries):
+                    raise APIConnectionError(f"Request failed: {exc}") from exc
+                time.sleep(C.backoff_seconds(attempt, None))
+                attempt += 1
+                continue
+
+            if C.should_retry(status, attempt, self.max_retries):
+                time.sleep(C.backoff_seconds(attempt, retry_after))
+                attempt += 1
+                continue
+            # non-retryable (or out of retries): raise a typed error
+            return C.parse_json_or_raise(status, resp.text, retry_after)
+
+    # -- API ---------------------------------------------------------------- #
+    def search(
+        self,
+        query: str,
+        *,
+        count: int = 10,
+        freshness: str = "all",
+        include_domains: Optional[Sequence[str]] = None,
+        exclude_domains: Optional[Sequence[str]] = None,
+        start_date: Optional[str] = None,
+        end_date: Optional[str] = None,
+        language: Optional[str] = None,
+        offset: int = 0,
+        include_excerpt: bool = False,
+        excerpt_chars: Optional[int] = None,
+    ) -> SearchResponse:
+        """Run a web search. Always costs 1 credit regardless of params."""
+        body = C.build_search_body(
+            query,
+            count=count,
+            freshness=freshness,
+            include_domains=include_domains,
+            exclude_domains=exclude_domains,
+            start_date=start_date,
+            end_date=end_date,
+            language=language,
+            offset=offset,
+            include_excerpt=include_excerpt,
+            excerpt_chars=excerpt_chars,
+        )
+        return SearchResponse.from_dict(self._post("/v1/search", body))
+
+    def fetch(
+        self, url_or_urls: Union[str, Sequence[str]]
+    ) -> Union[FetchResult, BatchFetchResponse]:
+        """Fetch indexed page content.
+
+        A single URL string returns a :class:`FetchResult`. A list returns a
+        :class:`BatchFetchResponse`. Lists longer than 50 are auto-chunked into
+        ≤50-URL calls, run sequentially with a small delay, and merged.
+        """
+        is_batch, urls, single = C.normalize_fetch_urls(url_or_urls)
+        if not is_batch:
+            data = self._post("/v1/fetch", {"url": single})
+            return FetchResult.from_dict(data)
+        return self._fetch_batch(urls)
+
+    def _fetch_batch(self, urls: Sequence[str]) -> BatchFetchResponse:
+        chunks = C.chunk_urls(urls)
+        merged_results: list[FetchResult] = []
+        merged_failed: list[FetchFailure] = []
+        remaining: Optional[int] = None
+        for i, chunk in enumerate(chunks):
+            if i > 0 and self.chunk_delay > 0:
+                time.sleep(self.chunk_delay)
+            data = self._post("/v1/fetch", {"urls": list(chunk)})
+            part = BatchFetchResponse.from_dict(data)
+            merged_results.extend(part.results)
+            merged_failed.extend(part.failed_results)
+            if part.remaining_quota is not None:
+                remaining = part.remaining_quota
+        return BatchFetchResponse(
+            results=merged_results,
+            failed_results=merged_failed,
+            count=len(merged_results),
+            remaining_quota=remaining,
+        )