devcopilot 0.2.0__py3-none-any.whl

api/web_tools/outbound.py

@@ -0,0 +1,278 @@
+"""Outbound HTTP for web_search / web_fetch (client, body caps, logging)."""
+
+from __future__ import annotations
+
+import asyncio
+import socket
+from collections.abc import AsyncIterator
+from urllib.parse import urljoin, urlparse
+
+import aiohttp
+import httpx
+from aiohttp import ClientSession, ClientTimeout, TCPConnector
+from aiohttp.abc import AbstractResolver, ResolveResult
+from loguru import logger
+
+from . import constants
+from .constants import (
+    _MAX_FETCH_CHARS,
+    _MAX_SEARCH_RESULTS,
+    _REDIRECT_RESPONSE_BODY_CAP_BYTES,
+    _REQUEST_TIMEOUT_S,
+    _WEB_FETCH_REDIRECT_STATUSES,
+    _WEB_TOOL_HTTP_HEADERS,
+)
+from .egress import (
+    WebFetchEgressPolicy,
+    WebFetchEgressViolation,
+    get_validated_stream_addrinfos_for_egress,
+)
+from .parsers import HTMLTextParser, SearchResultParser
+
+
+def _safe_public_host_for_logs(url: str) -> str:
+    host = urlparse(url).hostname or ""
+    return host[:253]
+
+
+def _log_web_tool_failure(
+    tool_name: str,
+    error: BaseException,
+    *,
+    fetch_url: str | None = None,
+) -> None:
+    exc_type = type(error).__name__
+    if isinstance(error, WebFetchEgressViolation):
+        host = _safe_public_host_for_logs(fetch_url) if fetch_url else ""
+        logger.warning(
+            "web_tool_egress_rejected tool={} exc_type={} host={!r}",
+            tool_name,
+            exc_type,
+            host,
+        )
+        return
+    if tool_name == "web_fetch" and fetch_url:
+        logger.warning(
+            "web_tool_failure tool={} exc_type={} host={!r}",
+            tool_name,
+            exc_type,
+            _safe_public_host_for_logs(fetch_url),
+        )
+    else:
+        logger.warning("web_tool_failure tool={} exc_type={}", tool_name, exc_type)
+
+
+def _web_tool_client_error_summary(
+    tool_name: str,
+    error: BaseException,
+    *,
+    verbose: bool,
+) -> str:
+    if verbose:
+        return f"{tool_name} failed: {type(error).__name__}"
+    return "Web tool request failed."
+
+
+async def _iter_response_body_under_cap(
+    response: httpx.Response, max_bytes: int
+) -> AsyncIterator[bytes]:
+    if max_bytes <= 0:
+        return
+    received = 0
+    async for chunk in response.aiter_bytes(chunk_size=65_536):
+        if received >= max_bytes:
+            break
+        remaining = max_bytes - received
+        if len(chunk) <= remaining:
+            received += len(chunk)
+            yield chunk
+            if received >= max_bytes:
+                break
+        else:
+            yield chunk[:remaining]
+            break
+
+
+async def _drain_response_body_capped(response: httpx.Response, max_bytes: int) -> None:
+    async for _ in _iter_response_body_under_cap(response, max_bytes):
+        pass
+
+
+async def _read_response_body_capped(response: httpx.Response, max_bytes: int) -> bytes:
+    return b"".join(
+        [piece async for piece in _iter_response_body_under_cap(response, max_bytes)]
+    )
+
+
+_NUMERIC_RESOLVE_FLAGS = socket.AI_NUMERICHOST | socket.AI_NUMERICSERV
+_NAME_RESOLVE_FLAGS = socket.NI_NUMERICHOST | socket.NI_NUMERICSERV
+
+
+def getaddrinfo_rows_to_resolve_results(
+    host: str, addrinfos: list[tuple]
+) -> list[ResolveResult]:
+    """Map :func:`socket.getaddrinfo` rows to aiohttp :class:`ResolveResult` (ThreadedResolver logic)."""
+    out: list[ResolveResult] = []
+    for family, _type, proto, _canon, sockaddr in addrinfos:
+        if family == socket.AF_INET6:
+            if len(sockaddr) < 3:
+                continue
+            if sockaddr[3]:
+                resolved_host, port = socket.getnameinfo(sockaddr, _NAME_RESOLVE_FLAGS)
+            else:
+                resolved_host, port = sockaddr[:2]
+        else:
+            assert family == socket.AF_INET, family
+            resolved_host, port = sockaddr[0], sockaddr[1]
+            resolved_host = str(resolved_host)
+            port = int(port)
+        out.append(
+            ResolveResult(
+                hostname=host,
+                host=resolved_host,
+                port=int(port),
+                family=family,
+                proto=proto,
+                flags=_NUMERIC_RESOLVE_FLAGS,
+            )
+        )
+    return out
+
+
+class _PinnedEgressStaticResolver(AbstractResolver):
+    """Return only pre-validated :class:`ResolveResult` for the outbound request."""
+
+    def __init__(self, results: list[ResolveResult]) -> None:
+        self._results = results
+
+    async def resolve(
+        self, host: str, port: int = 0, family: int = socket.AF_INET
+    ) -> list[ResolveResult]:
+        return self._results
+
+    async def close(self) -> None:  # pragma: no cover - aiohttp contract
+        return
+
+
+async def _read_aiohttp_body_capped(
+    response: aiohttp.ClientResponse, max_bytes: int
+) -> bytes:
+    received = 0
+    parts: list[bytes] = []
+    async for chunk in response.content.iter_chunked(65_536):
+        if received >= max_bytes:
+            break
+        remaining = max_bytes - received
+        if len(chunk) <= remaining:
+            received += len(chunk)
+            parts.append(chunk)
+        else:
+            parts.append(chunk[:remaining])
+            break
+    return b"".join(parts)
+
+
+async def _drain_aiohttp_body_capped(
+    response: aiohttp.ClientResponse, max_bytes: int
+) -> None:
+    if max_bytes <= 0:
+        return
+    received = 0
+    async for chunk in response.content.iter_chunked(65_536):
+        received += len(chunk)
+        if received >= max_bytes:
+            break
+
+
+async def _run_web_search(query: str) -> list[dict[str, str]]:
+    async with (
+        httpx.AsyncClient(
+            timeout=_REQUEST_TIMEOUT_S,
+            follow_redirects=True,
+            headers=_WEB_TOOL_HTTP_HEADERS,
+        ) as client,
+        client.stream(
+            "GET",
+            "https://lite.duckduckgo.com/lite/",
+            params={"q": query},
+        ) as response,
+    ):
+        response.raise_for_status()
+        body_bytes = await _read_response_body_capped(
+            response, constants._MAX_WEB_FETCH_RESPONSE_BYTES
+        )
+    text = body_bytes.decode("utf-8", errors="replace")
+    parser = SearchResultParser()
+    parser.feed(text)
+    return parser.results[:_MAX_SEARCH_RESULTS]
+
+
+async def _run_web_fetch(url: str, egress: WebFetchEgressPolicy) -> dict[str, str]:
+    """Fetch URL with manual redirects; each hop is DNS-pinned to validated addresses."""
+    current_url = url
+    redirect_hops = 0
+    timeout = ClientTimeout(total=_REQUEST_TIMEOUT_S)
+
+    while True:
+        addr_infos = await asyncio.to_thread(
+            get_validated_stream_addrinfos_for_egress, current_url, egress
+        )
+        host = urlparse(current_url).hostname or ""
+        results = getaddrinfo_rows_to_resolve_results(host, addr_infos)
+        resolver = _PinnedEgressStaticResolver(results)
+        connector = TCPConnector(
+            resolver=resolver,
+            force_close=True,
+        )
+        try:
+            async with (
+                ClientSession(
+                    timeout=timeout,
+                    headers=_WEB_TOOL_HTTP_HEADERS,
+                    connector=connector,
+                ) as session,
+                session.get(current_url, allow_redirects=False) as response,
+            ):
+                if response.status in _WEB_FETCH_REDIRECT_STATUSES:
+                    await _drain_aiohttp_body_capped(
+                        response, _REDIRECT_RESPONSE_BODY_CAP_BYTES
+                    )
+                    if redirect_hops >= constants._MAX_WEB_FETCH_REDIRECTS:
+                        raise WebFetchEgressViolation(
+                            "web_fetch exceeded maximum redirects "
+                            f"({constants._MAX_WEB_FETCH_REDIRECTS})"
+                        )
+                    location = response.headers.get("location")
+                    if not location or not location.strip():
+                        raise WebFetchEgressViolation(
+                            "web_fetch redirect response missing Location header"
+                        )
+                    current_url = urljoin(str(response.url), location.strip())
+                    redirect_hops += 1
+                    continue
+                response.raise_for_status()
+                content_type = response.headers.get("content-type", "text/plain")
+                final_url = str(response.url)
+                encoding = response.get_encoding() or "utf-8"
+                body_bytes = await _read_aiohttp_body_capped(
+                    response, constants._MAX_WEB_FETCH_RESPONSE_BYTES
+                )
+        finally:
+            await connector.close()
+
+        break
+
+    text = body_bytes.decode(encoding, errors="replace")
+    title = final_url
+    data = text
+    if "html" in content_type.lower():
+        parser = HTMLTextParser()
+        parser.feed(text)
+        title = parser.title or final_url
+        data = "\n".join(parser.text_parts)
+    return {
+        "url": final_url,
+        "title": title,
+        "media_type": "text/plain",
+        "data": data[:_MAX_FETCH_CHARS],
+    }

api/web_tools/parsers.py

@@ -0,0 +1,104 @@
+"""HTML parsing for web_search / web_fetch."""
+
+from __future__ import annotations
+
+import html
+import re
+from html.parser import HTMLParser
+from typing import Any
+from urllib.parse import parse_qs, unquote, urlparse
+
+
+class SearchResultParser(HTMLParser):
+    """DuckDuckGo lite HTML: extract result links and titles."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.results: list[dict[str, str]] = []
+        self._href: str | None = None
+        self._title_parts: list[str] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if tag != "a":
+            return
+        href = dict(attrs).get("href")
+        if not href or "uddg=" not in href:
+            return
+        parsed = urlparse(href)
+        query = parse_qs(parsed.query)
+        uddg = query.get("uddg", [""])[0]
+        if not uddg:
+            return
+        self._href = unquote(uddg)
+        self._title_parts = []
+
+    def handle_data(self, data: str) -> None:
+        if self._href is not None:
+            self._title_parts.append(data)
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag != "a" or self._href is None:
+            return
+        title = " ".join("".join(self._title_parts).split())
+        if title and not any(result["url"] == self._href for result in self.results):
+            self.results.append({"title": html.unescape(title), "url": self._href})
+        self._href = None
+        self._title_parts = []
+
+
+class HTMLTextParser(HTMLParser):
+    """Strip scripts/styles and collect visible text + title for fetch previews."""
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.title = ""
+        self.text_parts: list[str] = []
+        self._in_title = False
+        self._skip_depth = 0
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if tag in {"script", "style", "noscript"}:
+            self._skip_depth += 1
+        elif tag == "title":
+            self._in_title = True
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag in {"script", "style", "noscript"} and self._skip_depth:
+            self._skip_depth -= 1
+        elif tag == "title":
+            self._in_title = False
+
+    def handle_data(self, data: str) -> None:
+        text = " ".join(data.split())
+        if not text:
+            return
+        if self._in_title:
+            self.title = f"{self.title} {text}".strip()
+        elif not self._skip_depth:
+            self.text_parts.append(text)
+
+
+def content_text(content: Any) -> str:
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        parts = []
+        for item in content:
+            if isinstance(item, dict):
+                parts.append(str(item.get("text", "")))
+            else:
+                parts.append(str(getattr(item, "text", "")))
+        return "\n".join(part for part in parts if part)
+    return str(content)
+
+
+def extract_query(text: str) -> str:
+    match = re.search(r"query:\s*(.+)", text, flags=re.IGNORECASE | re.DOTALL)
+    if match:
+        return match.group(1).strip().strip("\"'")
+    return text.strip()
+
+
+def extract_url(text: str) -> str:
+    match = re.search(r"https?://\S+", text)
+    return match.group(0).rstrip(").,]") if match else text.strip()

api/web_tools/request.py

@@ -0,0 +1,87 @@
+"""Detect forced Anthropic web server tool requests."""
+
+from __future__ import annotations
+
+from api.models.anthropic import MessagesRequest, Tool
+
+
+def request_text(request: MessagesRequest) -> str:
+    """Join all user/assistant message content into one string for tool input parsing."""
+    from .parsers import content_text
+
+    return "\n".join(content_text(message.content) for message in request.messages)
+
+
+def forced_tool_turn_text(request: MessagesRequest) -> str:
+    """Text for parsing forced server-tool inputs: latest user turn only (avoids stale history)."""
+    if not request.messages:
+        return ""
+
+    from .parsers import content_text
+
+    for message in reversed(request.messages):
+        if message.role == "user":
+            return content_text(message.content)
+    return ""
+
+
+def forced_server_tool_name(request: MessagesRequest) -> str | None:
+    """Return web_search or web_fetch only when tool_choice forces that server tool."""
+    tc = request.tool_choice
+    if not isinstance(tc, dict):
+        return None
+    if tc.get("type") != "tool":
+        return None
+    name = tc.get("name")
+    if name in {"web_search", "web_fetch"}:
+        return str(name)
+    return None
+
+
+def has_tool_named(request: MessagesRequest, name: str) -> bool:
+    return any(tool.name == name for tool in request.tools or [])
+
+
+def is_web_server_tool_request(request: MessagesRequest) -> bool:
+    """True when the client forces a web server tool via tool_choice (not merely listed)."""
+    forced = forced_server_tool_name(request)
+    if forced is None:
+        return False
+    return has_tool_named(request, forced)
+
+
+def is_anthropic_server_tool_definition(tool: Tool) -> bool:
+    """Whether ``tool`` refers to an Anthropic server tool (web_search / web_fetch family)."""
+    name = (tool.name or "").strip()
+    if name in ("web_search", "web_fetch"):
+        return True
+    typ = tool.type
+    if isinstance(typ, str):
+        return typ.startswith("web_search") or typ.startswith("web_fetch")
+    return False
+
+
+def has_listed_anthropic_server_tools(request: MessagesRequest) -> bool:
+    """True when tools include web_search / web_fetch-style entries (listed, forced or not)."""
+    return any(is_anthropic_server_tool_definition(t) for t in (request.tools or []))
+
+
+def openai_chat_upstream_server_tool_error(
+    request: MessagesRequest, *, web_tools_enabled: bool
+) -> str | None:
+    """Return a user-facing error when OpenAI Chat upstream cannot satisfy server-tool semantics."""
+    forced = forced_server_tool_name(request)
+    if forced and not web_tools_enabled:
+        return (
+            f"tool_choice forces Anthropic server tool {forced!r}, but local web server tools are "
+            "disabled (ENABLE_WEB_SERVER_TOOLS=false). Enable them or use a native Anthropic "
+            "Messages transport (e.g. open_router, ollama, lmstudio)."
+        )
+    if not forced and has_listed_anthropic_server_tools(request):
+        return (
+            "OpenAI Chat upstreams cannot use listed Anthropic server tools "
+            "(web_search / web_fetch) without the local web server tool handler. Use a native "
+            "Anthropic transport, set ENABLE_WEB_SERVER_TOOLS=true and force the tool with "
+            "tool_choice, or remove these tools from the request."
+        )
+    return None

api/web_tools/streaming.py

@@ -0,0 +1,206 @@
+"""SSE streaming for local web_search / web_fetch server tool results."""
+
+from __future__ import annotations
+
+import uuid
+from collections.abc import AsyncIterator
+from datetime import UTC, datetime
+from typing import Any
+
+from api.models.anthropic import MessagesRequest
+from core.anthropic.server_tool_sse import (
+    SERVER_TOOL_USE,
+    WEB_FETCH_TOOL_ERROR,
+    WEB_FETCH_TOOL_RESULT,
+    WEB_SEARCH_TOOL_RESULT,
+    WEB_SEARCH_TOOL_RESULT_ERROR,
+)
+from core.anthropic.sse import format_sse_event
+
+from . import outbound
+from .constants import _MAX_FETCH_CHARS
+from .egress import WebFetchEgressPolicy
+from .parsers import extract_query, extract_url
+from .request import (
+    forced_server_tool_name,
+    forced_tool_turn_text,
+    has_tool_named,
+)
+
+
+def _search_summary(query: str, results: list[dict[str, str]]) -> str:
+    if not results:
+        return f"No web search results found for: {query}"
+    lines = [f"Search results for: {query}"]
+    for index, result in enumerate(results, start=1):
+        lines.append(f"{index}. {result['title']}\n{result['url']}")
+    return "\n\n".join(lines)
+
+
+async def stream_web_server_tool_response(
+    request: MessagesRequest,
+    input_tokens: int,
+    *,
+    web_fetch_egress: WebFetchEgressPolicy,
+    verbose_client_errors: bool = False,
+) -> AsyncIterator[str]:
+    """Stream a minimal Anthropic-shaped turn for forced `web_search` / `web_fetch` (local fallback).
+
+    When `ENABLE_WEB_SERVER_TOOLS` is on, this is a proxy-side execution path — not a full
+    hosted Anthropic citation or encrypted-content pipeline.
+    """
+    tool_name = forced_server_tool_name(request)
+    if tool_name is None or not has_tool_named(request, tool_name):
+        return
+
+    text = forced_tool_turn_text(request)
+    message_id = f"msg_{uuid.uuid4()}"
+    tool_id = f"srvtoolu_{uuid.uuid4().hex}"
+    usage_key = (
+        "web_search_requests" if tool_name == "web_search" else "web_fetch_requests"
+    )
+    tool_input = (
+        {"query": extract_query(text)}
+        if tool_name == "web_search"
+        else {"url": extract_url(text)}
+    )
+    _result_block_for_tool = {
+        "web_search": WEB_SEARCH_TOOL_RESULT,
+        "web_fetch": WEB_FETCH_TOOL_RESULT,
+    }
+    _error_payload_type_for_tool = {
+        "web_search": WEB_SEARCH_TOOL_RESULT_ERROR,
+        "web_fetch": WEB_FETCH_TOOL_ERROR,
+    }
+
+    yield format_sse_event(
+        "message_start",
+        {
+            "type": "message_start",
+            "message": {
+                "id": message_id,
+                "type": "message",
+                "role": "assistant",
+                "content": [],
+                "model": request.model,
+                "stop_reason": None,
+                "stop_sequence": None,
+                "usage": {"input_tokens": input_tokens, "output_tokens": 1},
+            },
+        },
+    )
+    yield format_sse_event(
+        "content_block_start",
+        {
+            "type": "content_block_start",
+            "index": 0,
+            "content_block": {
+                "type": SERVER_TOOL_USE,
+                "id": tool_id,
+                "name": tool_name,
+                "input": tool_input,
+            },
+        },
+    )
+    yield format_sse_event(
+        "content_block_stop", {"type": "content_block_stop", "index": 0}
+    )
+
+    try:
+        if tool_name == "web_search":
+            query = str(tool_input["query"])
+            results = await outbound._run_web_search(query)
+            result_content: Any = [
+                {
+                    "type": "web_search_result",
+                    "title": result["title"],
+                    "url": result["url"],
+                }
+                for result in results
+            ]
+            summary = _search_summary(query, results)
+            result_block_type = WEB_SEARCH_TOOL_RESULT
+        else:
+            fetched = await outbound._run_web_fetch(
+                str(tool_input["url"]), web_fetch_egress
+            )
+            result_content = {
+                "type": "web_fetch_result",
+                "url": fetched["url"],
+                "content": {
+                    "type": "document",
+                    "source": {
+                        "type": "text",
+                        "media_type": fetched["media_type"],
+                        "data": fetched["data"],
+                    },
+                    "title": fetched["title"],
+                    "citations": {"enabled": True},
+                },
+                "retrieved_at": datetime.now(UTC).isoformat(),
+            }
+            summary = fetched["data"][:_MAX_FETCH_CHARS]
+            result_block_type = WEB_FETCH_TOOL_RESULT
+    except Exception as error:
+        fetch_url = str(tool_input["url"]) if tool_name == "web_fetch" else None
+        outbound._log_web_tool_failure(tool_name, error, fetch_url=fetch_url)
+        result_block_type = _result_block_for_tool[tool_name]
+        result_content = {
+            "type": _error_payload_type_for_tool[tool_name],
+            "error_code": "unavailable",
+        }
+        summary = outbound._web_tool_client_error_summary(
+            tool_name, error, verbose=verbose_client_errors
+        )
+
+    output_tokens = max(1, len(summary) // 4)
+
+    yield format_sse_event(
+        "content_block_start",
+        {
+            "type": "content_block_start",
+            "index": 1,
+            "content_block": {
+                "type": result_block_type,
+                "tool_use_id": tool_id,
+                "content": result_content,
+            },
+        },
+    )
+    yield format_sse_event(
+        "content_block_stop", {"type": "content_block_stop", "index": 1}
+    )
+    # Model-facing summary: stream as normal text deltas (CLI/transcript code reads `text_delta`,
+    # not eager `text` on `content_block_start`).
+    yield format_sse_event(
+        "content_block_start",
+        {
+            "type": "content_block_start",
+            "index": 2,
+            "content_block": {"type": "text", "text": ""},
+        },
+    )
+    yield format_sse_event(
+        "content_block_delta",
+        {
+            "type": "content_block_delta",
+            "index": 2,
+            "delta": {"type": "text_delta", "text": summary},
+        },
+    )
+    yield format_sse_event(
+        "content_block_stop", {"type": "content_block_stop", "index": 2}
+    )
+    yield format_sse_event(
+        "message_delta",
+        {
+            "type": "message_delta",
+            "delta": {"stop_reason": "end_turn", "stop_sequence": None},
+            "usage": {
+                "input_tokens": input_tokens,
+                "output_tokens": output_tokens,
+                "server_tool_use": {usage_key: 1},
+            },
+        },
+    )
+    yield format_sse_event("message_stop", {"type": "message_stop"})

cli/__init__.py

@@ -0,0 +1,5 @@
+"""CLI integration for installed launchers and managed Claude Code."""
+
+from .managed import ManagedClaudeSession, ManagedClaudeSessionManager
+
+__all__ = ["ManagedClaudeSession", "ManagedClaudeSessionManager"]

cli/claude_env.py

@@ -0,0 +1,12 @@
+"""Shared Claude Code environment policy for FCC client surfaces."""
+
+from __future__ import annotations
+
+CLAUDE_CODE_AUTO_COMPACT_WINDOW = "190000"
+CLAUDE_NO_AUTH_SENTINEL = "fcc-no-auth"
+
+
+def claude_auth_token(auth_token: str) -> str:
+    """Return the Claude Code auth marker for proxy-auth or no-auth sessions."""
+
+    return auth_token.strip() or CLAUDE_NO_AUTH_SENTINEL