pi-web-access-py 0.1.0__py3-none-any.whl

pi_web_access/__init__.py

@@ -0,0 +1,45 @@
+"""pi-web-access — Web search and URL fetching extension for pi-python.
+
+Provides two tools:
+  - ``web_search`` — Search the web via Brave, Tavily, or SearXNG
+  - ``fetch_url`` — Fetch and extract text from a URL
+
+Install: ``pip install pi-web-access-py``
+
+Configuration (environment variables):
+  - ``BRAVE_API_KEY``  — Brave Search API key
+  - ``TAVILY_API_KEY`` — Tavily Search API key
+  - ``SEARXNG_URL``    — SearXNG instance URL
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from pi_agent_core.extensions import ExtensionAPI
+
+
+def activate(pi: ExtensionAPI) -> None:
+    """Extension entry point — called by the ExtensionLoader."""
+    from pi_web_access.fetch_url import create_fetch_url_tool
+    from pi_web_access.web_search import create_web_search_tool
+
+    pi.register_tool(create_web_search_tool())
+    pi.register_tool(create_fetch_url_tool())
+
+    # Passthrough: advertised for autocomplete but forwarded to LLM as a
+    # regular prompt so the model invokes the web_search / fetch_url tool.
+    _noop = lambda args: None  # noqa: E731
+    pi.register_command(
+        "web_search",
+        description="Search the web for real-time information",
+        handler=_noop,
+        passthrough=True,
+    )
+    pi.register_command(
+        "fetch_url",
+        description="Fetch and read the contents of a URL",
+        handler=_noop,
+        passthrough=True,
+    )

pi_web_access/fetch_url.py

@@ -0,0 +1,122 @@
+"""fetch_url tool — fetch and read the contents of a URL."""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+import httpx
+from pydantic import BaseModel, Field
+
+from pi_agent_core.coding_tools.truncate import DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, truncate_head
+from pi_agent_core.types import AgentToolResult
+
+
+class FetchUrlParams(BaseModel):
+    url: str = Field(description="The URL to fetch")
+    extract_text: bool = Field(
+        default=True,
+        description="Extract readable text from HTML (strip tags). Set false for raw content.",
+    )
+    max_length: int | None = Field(
+        default=None,
+        description="Maximum number of lines to return (default: 2000)",
+    )
+
+
+_TAG_RE = re.compile(r"<script[^>]*>.*?</script>|<style[^>]*>.*?</style>", re.DOTALL | re.I)
+_HTML_TAG_RE = re.compile(r"<[^>]+>")
+_WS_RE = re.compile(r"\n{3,}")
+
+
+def _simple_html_to_text(html: str) -> str:
+    """Lightweight HTML-to-text (no external deps)."""
+    text = _TAG_RE.sub("", html)
+    text = _HTML_TAG_RE.sub("", text)
+    text = text.replace("&amp;", "&").replace("&lt;", "<").replace("&gt;", ">")
+    text = text.replace("&quot;", '"').replace("&#39;", "'").replace("&nbsp;", " ")
+    text = _WS_RE.sub("\n\n", text)
+    return text.strip()
+
+
+def _extract_text(html: str) -> str:
+    """Try trafilatura first, fall back to simple tag stripping."""
+    try:
+        import trafilatura
+
+        result = trafilatura.extract(html, include_comments=False, include_tables=True)
+        if result:
+            return result
+    except ImportError:
+        pass
+    return _simple_html_to_text(html)
+
+
+async def fetch_url_execute(
+    tool_call_id: str,
+    params: Any,
+    signal: Any = None,
+    on_update: Any = None,
+) -> AgentToolResult:
+    try:
+        async with httpx.AsyncClient(
+            timeout=30,
+            follow_redirects=True,
+            headers={"User-Agent": "pi-python/0.1 (web-access extension)"},
+        ) as client:
+            resp = await client.get(params.url)
+            resp.raise_for_status()
+            raw = resp.text
+
+        if params.extract_text and "html" in resp.headers.get("content-type", "").lower():
+            text = _extract_text(raw)
+        else:
+            text = raw
+
+        max_lines = params.max_length or DEFAULT_MAX_LINES
+        result = truncate_head(text, max_lines=max_lines, max_bytes=DEFAULT_MAX_BYTES)
+
+        notice = ""
+        if result.truncated:
+            notice = (
+                f"\n[Showing {result.outputLines} of {result.totalLines} lines. "
+                f"Content truncated at {max_lines} lines.]"
+            )
+
+        return AgentToolResult(
+            content=[{"type": "text", "text": result.content + notice}],
+            details={
+                "url": params.url,
+                "statusCode": resp.status_code,
+                "contentType": resp.headers.get("content-type", ""),
+                "truncated": result.truncated,
+            },
+        )
+    except httpx.HTTPStatusError as e:
+        return AgentToolResult(
+            content=[
+                {"type": "text", "text": f"HTTP {e.response.status_code} fetching {params.url}"}
+            ]
+        )
+    except Exception as e:
+        return AgentToolResult(
+            content=[{"type": "text", "text": f"Failed to fetch {params.url}: {e}"}]
+        )
+
+
+def create_fetch_url_tool() -> Any:
+    """Return a ToolDefinition for the fetch_url tool."""
+    from pi_agent_core.extensions.types import ToolDefinition
+
+    return ToolDefinition(
+        name="fetch_url",
+        description="Fetch and read the contents of a URL. Extracts readable text from HTML pages.",
+        parameters=FetchUrlParams,
+        execute=fetch_url_execute,
+        label="Fetch URL",
+        prompt_snippet="Fetch and read the contents of a URL",
+        prompt_guidelines=[
+            "Use fetch_url to read the full content of a web page when you have a specific URL.",
+            "Combine with web_search: search first, then fetch relevant URLs for details.",
+        ],
+    )

pi_web_access/providers/__init__.py

@@ -0,0 +1,7 @@
+"""Search provider adapters for pi-web-access."""
+
+from pi_web_access.providers.brave import BraveProvider
+from pi_web_access.providers.searxng import SearXNGProvider
+from pi_web_access.providers.tavily import TavilyProvider
+
+__all__ = ["BraveProvider", "SearXNGProvider", "TavilyProvider"]

pi_web_access/providers/brave.py

@@ -0,0 +1,49 @@
+"""Brave Search API provider."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+
+
+@dataclass
+class SearchResult:
+    title: str
+    url: str
+    snippet: str
+
+
+class BraveProvider:
+    """Brave Search API (https://api.search.brave.com)."""
+
+    BASE_URL = "https://api.search.brave.com/res/v1/web/search"
+
+    def __init__(self, api_key: str) -> None:
+        self.api_key = api_key
+
+    async def search(self, query: str, max_results: int = 5) -> list[SearchResult]:
+        async with httpx.AsyncClient(timeout=30) as client:
+            resp = await client.get(
+                self.BASE_URL,
+                params={"q": query, "count": min(max_results, 20)},
+                headers={
+                    "Accept": "application/json",
+                    "Accept-Encoding": "gzip",
+                    "X-Subscription-Token": self.api_key,
+                },
+            )
+            resp.raise_for_status()
+            data: dict[str, Any] = resp.json()
+
+        results: list[SearchResult] = []
+        for item in (data.get("web", {}).get("results") or [])[:max_results]:
+            results.append(
+                SearchResult(
+                    title=item.get("title", ""),
+                    url=item.get("url", ""),
+                    snippet=item.get("description", ""),
+                )
+            )
+        return results

pi_web_access/providers/searxng.py

@@ -0,0 +1,37 @@
+"""SearXNG self-hosted provider."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import httpx
+
+from pi_web_access.providers.brave import SearchResult
+
+
+class SearXNGProvider:
+    """SearXNG self-hosted instance (JSON API)."""
+
+    def __init__(self, base_url: str) -> None:
+        self.base_url = base_url.rstrip("/")
+
+    async def search(self, query: str, max_results: int = 5) -> list[SearchResult]:
+        url = f"{self.base_url}/search"
+        async with httpx.AsyncClient(timeout=30) as client:
+            resp = await client.get(
+                url,
+                params={"q": query, "format": "json", "pageno": 1},
+            )
+            resp.raise_for_status()
+            data: dict[str, Any] = resp.json()
+
+        results: list[SearchResult] = []
+        for item in (data.get("results") or [])[:max_results]:
+            results.append(
+                SearchResult(
+                    title=item.get("title", ""),
+                    url=item.get("url", ""),
+                    snippet=item.get("content", ""),
+                )
+            )
+        return results

pi_web_access/providers/tavily.py

@@ -0,0 +1,43 @@
+"""Tavily Search API provider."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import httpx
+
+from pi_web_access.providers.brave import SearchResult
+
+
+class TavilyProvider:
+    """Tavily Search API (https://api.tavily.com)."""
+
+    BASE_URL = "https://api.tavily.com/search"
+
+    def __init__(self, api_key: str) -> None:
+        self.api_key = api_key
+
+    async def search(self, query: str, max_results: int = 5) -> list[SearchResult]:
+        async with httpx.AsyncClient(timeout=30) as client:
+            resp = await client.post(
+                self.BASE_URL,
+                json={
+                    "api_key": self.api_key,
+                    "query": query,
+                    "max_results": min(max_results, 20),
+                    "include_answer": False,
+                },
+            )
+            resp.raise_for_status()
+            data: dict[str, Any] = resp.json()
+
+        results: list[SearchResult] = []
+        for item in (data.get("results") or [])[:max_results]:
+            results.append(
+                SearchResult(
+                    title=item.get("title", ""),
+                    url=item.get("url", ""),
+                    snippet=item.get("content", ""),
+                )
+            )
+        return results

pi_web_access/web_search.py

@@ -0,0 +1,144 @@
+"""web_search tool — search the web for real-time information."""
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from pi_agent_core.types import AgentToolResult
+
+
+class WebSearchParams(BaseModel):
+    query: str = Field(description="The search query")
+    provider: str | None = Field(
+        default=None,
+        description="Search provider: 'brave', 'tavily', or 'searxng'. Auto-detected if omitted.",
+    )
+    max_results: int = Field(
+        default=5,
+        ge=1,
+        le=20,
+        description="Maximum number of results to return",
+    )
+
+
+def _detect_provider() -> tuple[str, str]:
+    """Auto-detect search provider from environment variables.
+
+    Returns (provider_name, credential).
+    """
+    brave_key = os.environ.get("BRAVE_API_KEY")
+    if brave_key:
+        return "brave", brave_key
+
+    tavily_key = os.environ.get("TAVILY_API_KEY")
+    if tavily_key:
+        return "tavily", tavily_key
+
+    searxng_url = os.environ.get("SEARXNG_URL")
+    if searxng_url:
+        return "searxng", searxng_url
+
+    return "", ""
+
+
+async def web_search_execute(
+    tool_call_id: str,
+    params: Any,
+    signal: Any = None,
+    on_update: Any = None,
+) -> AgentToolResult:
+    provider_name = params.provider
+    credential = ""
+
+    if provider_name:
+        if provider_name == "brave":
+            credential = os.environ.get("BRAVE_API_KEY", "")
+        elif provider_name == "tavily":
+            credential = os.environ.get("TAVILY_API_KEY", "")
+        elif provider_name == "searxng":
+            credential = os.environ.get("SEARXNG_URL", "")
+        else:
+            return AgentToolResult(
+                content=[{"type": "text", "text": f"Unknown provider: {provider_name}"}]
+            )
+    else:
+        provider_name, credential = _detect_provider()
+
+    if not provider_name or not credential:
+        return AgentToolResult(
+            content=[
+                {
+                    "type": "text",
+                    "text": (
+                        "No search provider configured. Set one of these environment variables:\n"
+                        "  BRAVE_API_KEY   — Brave Search API key\n"
+                        "  TAVILY_API_KEY  — Tavily Search API key\n"
+                        "  SEARXNG_URL     — SearXNG instance URL (e.g. http://localhost:8080)"
+                    ),
+                }
+            ]
+        )
+
+    try:
+        if provider_name == "brave":
+            from pi_web_access.providers.brave import BraveProvider
+
+            provider = BraveProvider(credential)
+        elif provider_name == "tavily":
+            from pi_web_access.providers.tavily import TavilyProvider
+
+            provider = TavilyProvider(credential)
+        elif provider_name == "searxng":
+            from pi_web_access.providers.searxng import SearXNGProvider
+
+            provider = SearXNGProvider(credential)
+        else:
+            return AgentToolResult(
+                content=[{"type": "text", "text": f"Unknown provider: {provider_name}"}]
+            )
+
+        results = await provider.search(params.query, params.max_results)
+
+        if not results:
+            return AgentToolResult(
+                content=[{"type": "text", "text": f"No results found for: {params.query}"}]
+            )
+
+        lines = [f"Search results for: {params.query}\n"]
+        for i, r in enumerate(results, 1):
+            lines.append(f"{i}. {r.title}")
+            lines.append(f"   {r.url}")
+            if r.snippet:
+                lines.append(f"   {r.snippet}")
+            lines.append("")
+
+        return AgentToolResult(
+            content=[{"type": "text", "text": "\n".join(lines)}],
+            details={"provider": provider_name, "resultCount": len(results)},
+        )
+    except Exception as e:
+        return AgentToolResult(
+            content=[{"type": "text", "text": f"Search failed ({provider_name}): {e}"}]
+        )
+
+
+def create_web_search_tool() -> dict[str, Any]:
+    """Return a ToolDefinition-compatible dict for the web_search tool."""
+    from pi_agent_core.extensions.types import ToolDefinition
+
+    return ToolDefinition(
+        name="web_search",
+        description="Search the web for real-time information using Brave, Tavily, or SearXNG",
+        parameters=WebSearchParams,
+        execute=web_search_execute,
+        label="Web Search",
+        prompt_snippet="Search the web for real-time information",
+        prompt_guidelines=[
+            "Use web_search when the user asks about current events, recent news, "
+            "or information that may not be in your training data.",
+            "Prefer web_search over guessing when you are unsure about facts.",
+        ],
+    )

pi_web_access_py-0.1.0.dist-info/METADATA

@@ -0,0 +1,43 @@
+Metadata-Version: 2.5
+Name: pi-web-access-py
+Version: 0.1.0
+Summary: Web search and URL fetching extension for pi-python (port of pi-web-access)
+Project-URL: Homepage, https://github.com/zy1233/pi-python
+Project-URL: Repository, https://github.com/zy1233/pi-python
+Author-email: zy1233 <zy1233@users.noreply.github.com>
+License-Expression: MIT
+Keywords: agent,extension,fetch,pi,web-search
+Requires-Python: >=3.11
+Requires-Dist: httpx>=0.27
+Requires-Dist: pi-agent-core-lc>=0.3.0
+Provides-Extra: brave
+Provides-Extra: readability
+Requires-Dist: trafilatura>=1.12; extra == 'readability'
+Provides-Extra: tavily
+Requires-Dist: tavily-python>=0.5; extra == 'tavily'
+Description-Content-Type: text/markdown
+
+# pi-web-access-py
+
+Web search and URL fetching extension for [pi-python](https://github.com/zy1233/pi-python).
+
+Python port of [pi-web-access](https://www.npmjs.com/package/pi-web-access).
+
+## Install
+
+```bash
+pip install pi-web-access-py
+```
+
+## Configuration
+
+Set one of these environment variables:
+
+- `BRAVE_API_KEY` — Brave Search API key
+- `TAVILY_API_KEY` — Tavily Search API key
+- `SEARXNG_URL` — SearXNG instance URL
+
+## Tools
+
+- **web_search** — Search the web for real-time information
+- **fetch_url** — Fetch and extract text from a URL

pi_web_access_py-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+pi_web_access/__init__.py,sha256=uiTR0OuIX_FJ5rMyuvyIYEOCb0btw0A_tK_net1Ag5Y,1443
+pi_web_access/fetch_url.py,sha256=kwpoZRUffOKdLUjCHR_LWr3J3zJLBD6hWqYI6HbO4gA,4057
+pi_web_access/web_search.py,sha256=IW5ML_MUOQDA4MyD0nP1s1YnVf5vYawrJyYq9ynDhvg,4743
+pi_web_access/providers/__init__.py,sha256=bJGNrTBmDQsuAB5ru1suNTiBu8vP1thrMLfnVGpmbC0,291
+pi_web_access/providers/brave.py,sha256=sjE6N2VEgiOMmS47zPyr24skPqdi861DCgeY4niokaA,1406
+pi_web_access/providers/searxng.py,sha256=czOelkddMLaXWmbrzD6Vwa6ybfoB-Vw84dFV-1019Io,1108
+pi_web_access/providers/tavily.py,sha256=NgVx2V_zxkxeLoiiw-gOp0bX8Py2lHw8BcDrKGcO_jU,1269
+pi_web_access_py-0.1.0.dist-info/METADATA,sha256=qCT6zmlwZsUo4lZTk8HaVU8wi4KWS0bePf3TAz6258o,1264
+pi_web_access_py-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+pi_web_access_py-0.1.0.dist-info/entry_points.txt,sha256=i82VtScN0CIfgaXCe8naMfzrtDacLlItxaBFCKnaz4k,61
+pi_web_access_py-0.1.0.dist-info/RECORD,,

pi_web_access_py-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

pi_web_access_py-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[pi_agent.extensions]
+pi-web-access = pi_web_access:activate