agentengine-sdk-python 0.6.2__py3-none-any.whl → 0.6.9__py3-none-any.whl

ksadk/toolsets/web.py

@@ -0,0 +1,266 @@
+from __future__ import annotations
+
+import html
+import ipaddress
+import json
+import os
+import socket
+from typing import Any
+from urllib.parse import quote_plus, urljoin, urlparse
+
+import httpx
+from bs4 import BeautifulSoup
+
+from ksadk.tools.gateway import ToolPolicy, default_tool_gateway
+from ksadk.tools.result_budget import budget_tool_output, default_tool_result_budget
+from ksadk.toolsets._langchain import as_tool
+
+
+_WEB_TOOL_POLICIES = {
+    "web_fetch": ToolPolicy(risk_level="low"),
+    "web_search": ToolPolicy(risk_level="low"),
+}
+
+
+def _gateway():
+    return default_tool_gateway(_WEB_TOOL_POLICIES)
+
+
+def web_fetch(url: str, max_chars: int | None = None, timeout: int = 30) -> dict:
+    """Fetch a public HTTP(S) URL and return sanitized text content."""
+
+    return _gateway().invoke("web_fetch", _web_fetch_impl, url, max_chars, timeout)
+
+
+def _web_fetch_impl(url: str, max_chars: int | None = None, timeout: int = 30) -> dict:
+    current_url = str(url or "").strip()
+    if not current_url:
+        return {"ok": False, "error_type": "url_required", "error_message": "url is required"}
+    for _ in range(5):
+        blocked = _validate_public_http_url(current_url)
+        if blocked:
+            return blocked
+        try:
+            with httpx.Client(follow_redirects=False, timeout=timeout) as client:
+                response = client.get(current_url)
+        except httpx.TimeoutException:
+            return {"ok": False, "error_type": "timeout", "error_message": "web_fetch timed out", "url": current_url}
+        except httpx.HTTPError as exc:
+            return {"ok": False, "error_type": type(exc).__name__, "error_message": str(exc), "url": current_url}
+        if response.is_redirect and response.headers.get("location"):
+            current_url = urljoin(current_url, str(response.headers["location"]))
+            continue
+        content_type = response.headers.get("content-type", "")
+        text = _html_to_text(response.text) if "html" in content_type.lower() else response.text
+        limit = max_chars or 50_000
+        budgeted = budget_tool_output(
+            tool_name="web_fetch",
+            field_name="text",
+            value=text,
+            metadata={"tool_use_id": "web_fetch"},
+            budget=default_tool_result_budget(max_chars=limit, preview_chars=limit, persist_threshold_chars=limit),
+        )
+        return {
+            "ok": True,
+            "url": url,
+            "final_url": str(response.url),
+            "status_code": response.status_code,
+            "content_type": content_type,
+            **budgeted,
+        }
+    return {"ok": False, "error_type": "too_many_redirects", "error_message": "web_fetch exceeded redirect limit", "url": url}
+
+
+def web_search(query: str, max_results: int = 5, recency_days: int | None = None) -> dict:
+    """Search the web and return title/url/snippet results without fetching full pages."""
+
+    return _gateway().invoke("web_search", _web_search_impl, query, max_results, recency_days)
+
+
+def _web_search_impl(query: str, max_results: int = 5, recency_days: int | None = None) -> dict:
+    text = str(query or "").strip()
+    if not text:
+        return {"ok": False, "error_type": "query_required", "error_message": "query is required"}
+    provider = _web_search_provider()
+    if not provider:
+        return {"ok": False, "error_type": "provider_not_configured", "error_message": "web search provider is not configured"}
+    if provider == "fake":
+        return {
+            "ok": True,
+            "query": text,
+            "provider": "fake",
+            "results": [
+                {
+                    "title": f"Fake result for {text}",
+                    "url": f"https://example.com/search?q={quote_plus(text)}",
+                    "snippet": "Fake search result generated by ksadk test provider.",
+                    "rank": 1,
+                    "provider": "fake",
+                }
+            ][: max(1, int(max_results or 5))],
+        }
+    if provider == "http":
+        return _web_search_http(text, max_results=max_results, recency_days=recency_days)
+    if provider == "ksyun":
+        return _web_search_ksyun(text, max_results=max_results)
+    return {"ok": False, "error_type": "provider_not_configured", "error_message": f"unsupported web search provider: {provider}"}
+
+
+def get_web_tools() -> list:
+    return [as_tool(web_fetch), as_tool(web_search)]
+
+
+def _web_search_provider() -> str:
+    return (
+        os.environ.get("KSADK_WEB_SEARCH_PROVIDER")
+        or os.environ.get("OPENCLAW_WEB_SEARCH_PROVIDER")
+        or ""
+    ).strip().lower()
+
+
+def _web_search_http(query: str, *, max_results: int = 5, recency_days: int | None = None) -> dict[str, Any]:
+    base_url = str(os.environ.get("KSADK_WEB_SEARCH_BASE_URL") or os.environ.get("OPENCLAW_WEB_SEARCH_BASE_URL") or "").strip()
+    if not base_url:
+        return {"ok": False, "error_type": "provider_not_configured", "error_message": "web search base URL is not configured"}
+    api_key = str(os.environ.get("KSADK_WEB_SEARCH_API_KEY") or os.environ.get("OPENCLAW_WEB_SEARCH_API_KEY") or "").strip()
+    params: dict[str, Any] = {"q": query, "max_results": max(1, int(max_results or 5))}
+    if recency_days is not None:
+        params["recency_days"] = int(recency_days)
+    headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}
+    try:
+        with httpx.Client(timeout=30) as client:
+            response = client.get(base_url, params=params, headers=headers)
+        response.raise_for_status()
+        payload = response.json()
+    except httpx.TimeoutException:
+        return {"ok": False, "error_type": "timeout", "error_message": "web_search timed out"}
+    except Exception as exc:
+        return {"ok": False, "error_type": type(exc).__name__, "error_message": str(exc)}
+    raw_results = payload.get("results") if isinstance(payload, dict) else None
+    if not isinstance(raw_results, list):
+        raw_results = payload.get("data") if isinstance(payload, dict) else []
+    results = []
+    for index, item in enumerate(raw_results or [], start=1):
+        if not isinstance(item, dict):
+            continue
+        results.append(
+            {
+                "title": str(item.get("title") or ""),
+                "url": str(item.get("url") or item.get("link") or ""),
+                "snippet": str(item.get("snippet") or item.get("summary") or ""),
+                "rank": index,
+                "provider": "http",
+            }
+        )
+        if len(results) >= params["max_results"]:
+            break
+    return {"ok": True, "query": query, "provider": "http", "results": results}
+
+
+def _ksyun_search_endpoint() -> str:
+    return (
+        os.environ.get("KSADK_WEB_SEARCH_BASE_URL")
+        or os.environ.get("OPENCLAW_WEB_SEARCH_BASE_URL")
+        or "https://search.aipro.ksyun.com/v1/aisearch/search"
+    ).strip()
+
+
+def _ksyun_search_api_key() -> str:
+    # 金山云 AI搜索与 metaso 同属星流平台，复用 KSADK_MCP_KEY / KSC_AIPRO_API_KEY 凭证。
+    for key in (
+        "KSADK_WEB_SEARCH_API_KEY",
+        "OPENCLAW_WEB_SEARCH_API_KEY",
+        "KSADK_MCP_KEY",
+        "KSC_AIPRO_API_KEY",
+        "KSC_AIPRO_APIKEY",
+        "AIPRO_API_KEY",
+    ):
+        value = os.environ.get(key, "").strip()
+        if value:
+            return value
+    return ""
+
+
+def _ksyun_search_scope() -> str:
+    # 金山云 AI搜索 scope：webpage/document/scholar/podcast/video，默认 webpage。
+    return (os.environ.get("KSADK_WEB_SEARCH_SCOPE") or "webpage").strip().lower() or "webpage"
+
+
+def _web_search_ksyun(query: str, *, max_results: int = 5) -> dict[str, Any]:
+    endpoint = _ksyun_search_endpoint()
+    api_key = _ksyun_search_api_key()
+    if not api_key:
+        return {"ok": False, "error_type": "provider_not_configured", "error_message": "ksyun web search api key is not configured (KSADK_WEB_SEARCH_API_KEY / KSADK_MCP_KEY / KSC_AIPRO_API_KEY)"}
+    size = max(1, min(int(max_results or 5), 200))
+    headers = {"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"}
+    payload_body = {"q": query, "scope": _ksyun_search_scope(), "size": size}
+    try:
+        with httpx.Client(timeout=30) as client:
+            response = client.post(endpoint, headers=headers, json=payload_body)
+        response.raise_for_status()
+        payload = response.json()
+    except httpx.TimeoutException:
+        return {"ok": False, "error_type": "timeout", "error_message": "web_search timed out"}
+    except Exception as exc:
+        return {"ok": False, "error_type": type(exc).__name__, "error_message": str(exc)}
+    raw_results = payload.get("webpages") if isinstance(payload, dict) else None
+    if not isinstance(raw_results, list):
+        # 兼容返回 results/data 的情形。
+        raw_results = payload.get("results") if isinstance(payload, dict) else None
+    if not isinstance(raw_results, list):
+        raw_results = payload.get("data") if isinstance(payload, dict) else []
+    results = []
+    for index, item in enumerate(raw_results or [], start=1):
+        if not isinstance(item, dict):
+            continue
+        results.append(
+            {
+                "title": str(item.get("title") or ""),
+                "url": str(item.get("url") or item.get("link") or ""),
+                "snippet": str(item.get("snippet") or item.get("summary") or ""),
+                "date": str(item.get("date") or item.get("published_at") or ""),
+                "rank": index,
+                "provider": "ksyun",
+            }
+        )
+        if len(results) >= size:
+            break
+    return {"ok": True, "query": query, "provider": "ksyun", "results": results}
+
+
+def _validate_public_http_url(url: str) -> dict | None:
+    parsed = urlparse(url)
+    if parsed.scheme not in {"http", "https"}:
+        return {"ok": False, "error_type": "unsupported_scheme", "error_message": "only http and https URLs are supported", "url": url}
+    if not parsed.hostname:
+        return {"ok": False, "error_type": "invalid_url", "error_message": "URL host is required", "url": url}
+    policy = _ssrf_policy()
+    if policy.get("allow_private") is True:
+        return None
+    try:
+        addresses = socket.getaddrinfo(parsed.hostname, parsed.port or (443 if parsed.scheme == "https" else 80), type=socket.SOCK_STREAM)
+    except socket.gaierror as exc:
+        return {"ok": False, "error_type": "dns_resolution_failed", "error_message": str(exc), "url": url}
+    for address in addresses:
+        ip = ipaddress.ip_address(address[4][0])
+        if ip.is_loopback or ip.is_private or ip.is_link_local or ip.is_multicast or ip.is_reserved:
+            return {"ok": False, "error_type": "blocked_by_ssrf_policy", "error_message": f"blocked non-public address: {ip}", "url": url}
+    return None
+
+
+def _ssrf_policy() -> dict:
+    raw = os.environ.get("KSADK_WEB_SSRF_POLICY_JSON") or os.environ.get("OPENCLAW_BROWSER_SSRF_POLICY_JSON") or ""
+    if not raw:
+        return {}
+    try:
+        parsed = json.loads(raw)
+    except json.JSONDecodeError:
+        return {}
+    return parsed if isinstance(parsed, dict) else {}
+
+
+def _html_to_text(value: str) -> str:
+    soup = BeautifulSoup(value or "", "html.parser")
+    for element in soup(["script", "style", "noscript"]):
+        element.decompose()
+    return html.unescape(" ".join(soup.get_text(" ").split()))