ccproxy-api 0.1.6__py3-none-any.whl → 0.2.0__py3-none-any.whl

ccproxy/streaming/handler.py

@@ -0,0 +1,117 @@
+"""Streaming request handler for SSE and chunked responses."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+import httpx
+import structlog
+
+from ccproxy.core.plugins.hooks import HookManager
+from ccproxy.core.request_context import RequestContext
+from ccproxy.services.handler_config import HandlerConfig
+from ccproxy.streaming.deferred import DeferredStreaming
+
+
+logger = structlog.get_logger(__name__)
+
+
+class StreamingHandler:
+    """Manages streaming request processing with header preservation and SSE adaptation."""
+
+    def __init__(
+        self,
+        hook_manager: HookManager | None = None,
+    ) -> None:
+        """Initialize with hook manager for stream events.
+
+        Args:
+            hook_manager: Optional hook manager for emitting stream events
+        """
+        self.hook_manager = hook_manager
+
+    def should_stream_response(self, headers: dict[str, str]) -> bool:
+        """Detect streaming intent from request headers.
+
+        - Prefer client `Accept: text/event-stream`
+        - Fallback to provider-style `Content-Type: text/event-stream` (rare for requests)
+        - Case-insensitive checks
+        """
+        accept = str(headers.get("accept", "")).lower()
+        if "text/event-stream" in accept:
+            return True
+
+        content_type = str(headers.get("content-type", "")).lower()
+        return "text/event-stream" in content_type
+
+    async def should_stream(
+        self, request_body: bytes, handler_config: HandlerConfig
+    ) -> bool:
+        """Check if request body has stream:true flag.
+
+        - Returns False if provider doesn't support streaming
+        - Parses JSON body for 'stream' field
+        - Handles parse errors gracefully
+        """
+        if not handler_config.supports_streaming:
+            return False
+
+        try:
+            data = json.loads(request_body)
+            return data.get("stream", False) is True
+        except (json.JSONDecodeError, TypeError):
+            return False
+
+    async def handle_streaming_request(
+        self,
+        method: str,
+        url: str,
+        headers: dict[str, str],
+        body: bytes,
+        handler_config: HandlerConfig,
+        request_context: RequestContext,
+        on_headers: Any | None = None,
+        client_config: dict[str, Any] | None = None,
+        client: httpx.AsyncClient | None = None,
+    ) -> DeferredStreaming:
+        """Create a deferred streaming response that preserves headers.
+
+        This always returns a DeferredStreaming response which:
+        - Defers the actual HTTP request until FastAPI sends the response
+        - Captures all upstream headers correctly
+        - Supports SSE processing through handler_config
+        - Provides request tracing and metrics
+        """
+
+        # Use provided client or create a short-lived one
+        owns_client = False
+        if client is None:
+            client = httpx.AsyncClient(**(client_config or {}))
+            owns_client = True
+
+        # Log that we're creating a deferred response
+        logger.debug(
+            "streaming_handler_creating_deferred_response",
+            url=url,
+            method=method,
+            has_sse_adapter=bool(handler_config.response_adapter),
+            adapter_type=type(handler_config.response_adapter).__name__
+            if handler_config.response_adapter
+            else None,
+        )
+
+        # Return the deferred response with format adapter from handler config
+        return DeferredStreaming(
+            method=method,
+            url=url,
+            headers=headers,
+            body=body,
+            client=client,
+            media_type="text/event-stream; charset=utf-8",
+            handler_config=handler_config,  # Contains format adapter if needed
+            request_context=request_context,
+            hook_manager=self.hook_manager,
+            on_headers=on_headers,
+            close_client_on_finish=owns_client,
+        )

ccproxy/streaming/interfaces.py

@@ -0,0 +1,77 @@
+"""Streaming interfaces for provider implementations.
+
+This module defines interfaces that providers can implement to extend
+streaming functionality without coupling core code to specific providers.
+"""
+
+from typing import Protocol
+
+from typing_extensions import TypedDict
+
+
+class StreamingMetrics(TypedDict, total=False):
+    """Standard streaming metrics structure."""
+
+    tokens_input: int | None
+    tokens_output: int | None
+    cache_read_tokens: int | None
+    cache_write_tokens: int | None
+    cost_usd: float | None
+
+
+class IStreamingMetricsCollector(Protocol):
+    """Interface for provider-specific streaming metrics collection.
+
+    Providers implement this interface to extract token usage and other
+    metrics from their specific streaming response formats.
+    """
+
+    def process_chunk(self, chunk_str: str) -> bool:
+        """Process a streaming chunk to extract metrics.
+
+        Args:
+            chunk_str: Raw chunk string from streaming response
+
+        Returns:
+            True if this was the final chunk with complete metrics, False otherwise
+        """
+        ...
+
+    def process_raw_chunk(self, chunk_str: str) -> bool:
+        """Process a raw provider chunk before any format conversion.
+
+        This method is called with chunks in the provider's native format,
+        before any OpenAI/Anthropic format conversion happens.
+
+        Args:
+            chunk_str: Raw chunk string in provider's native format
+
+        Returns:
+            True if this was the final chunk with complete metrics, False otherwise
+        """
+        ...
+
+    def process_converted_chunk(self, chunk_str: str) -> bool:
+        """Process a chunk after format conversion.
+
+        This method is called with chunks after they've been converted
+        to a different format (e.g., OpenAI format).
+
+        Args:
+            chunk_str: Chunk string after format conversion
+
+        Returns:
+            True if this was the final chunk with complete metrics, False otherwise
+        """
+        ...
+
+    def get_metrics(self) -> StreamingMetrics:
+        """Get the collected metrics.
+
+        Returns:
+            Dictionary with provider-specific metrics (tokens, costs, etc.)
+        """
+        ...
+
+
+# Moved StreamingConfigurable to ccproxy.core.interfaces to avoid circular imports

ccproxy/streaming/simple_adapter.py

@@ -0,0 +1,39 @@
+"""Simplified streaming adapter that bypasses complex type conversions.
+
+This adapter provides a direct dict-based interface for streaming without
+the complexity of the shim layer.
+"""
+
+from collections.abc import AsyncGenerator, AsyncIterator
+from typing import Any
+
+
+class SimpleStreamingAdapter:
+    """Simple adapter for streaming responses that works directly with dicts."""
+
+    def __init__(self, name: str = "simple_streaming"):
+        """Initialize the simple adapter."""
+        self.name = name
+
+    async def adapt_request(self, request: dict[str, Any]) -> dict[str, Any]:
+        """Pass through request - no adaptation needed for streaming."""
+        return request
+
+    async def adapt_response(self, response: dict[str, Any]) -> dict[str, Any]:
+        """Pass through response - no adaptation needed for streaming."""
+        return response
+
+    def adapt_stream(
+        self, stream: AsyncIterator[dict[str, Any]]
+    ) -> AsyncGenerator[dict[str, Any], None]:
+        """Pass through stream - no adaptation needed for simple streaming."""
+
+        async def passthrough_stream() -> AsyncGenerator[dict[str, Any], None]:
+            async for chunk in stream:
+                yield chunk
+
+        return passthrough_stream()
+
+    async def adapt_error(self, error: dict[str, Any]) -> dict[str, Any]:
+        """Pass through error - no adaptation needed."""
+        return error

ccproxy/streaming/sse.py

@@ -0,0 +1,109 @@
+"""Shared helpers for serializing JSON streams into SSE messages."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import AsyncGenerator, AsyncIterator
+from typing import Any
+
+from ccproxy.core.logging import get_logger
+from ccproxy.llms.streaming import AnthropicSSEFormatter
+
+
+logger = get_logger(__name__)
+
+
+async def serialize_json_to_sse_stream(
+    json_stream: AsyncIterator[Any],
+    *,
+    include_done: bool = True,
+    request_context: Any | None = None,
+) -> AsyncGenerator[bytes, None]:
+    """Serialize JSON-like stream items into SSE-compliant bytes.
+
+    This matches the behaviour previously implemented in
+    ``DeferredStreaming._serialize_json_to_sse_stream`` and is shared by
+    SDK and HTTP-based providers alike.
+
+    Args:
+        json_stream: Async iterator yielding dict-like SSE payloads (or
+            objects with ``model_dump``/``dict``).
+        include_done: Whether to append the ``data: [DONE]`` sentinel at
+            the end of the stream.
+        request_context: Optional request context for logging (expects a
+            ``request_id`` attribute when present).
+    """
+
+    formatter = AnthropicSSEFormatter()
+    request_id = None
+    if request_context and hasattr(request_context, "request_id"):
+        request_id = request_context.request_id
+
+    chunk_count = 0
+    anthropic_chunks = 0
+    openai_chunks = 0
+
+    async for json_obj in json_stream:
+        chunk_count += 1
+
+        # Normalise the payload to a dict
+        if hasattr(json_obj, "model_dump") and callable(json_obj.model_dump):
+            json_obj = json_obj.model_dump()
+        elif hasattr(json_obj, "dict") and callable(json_obj.dict):
+            json_obj = json_obj.dict()
+
+        if not isinstance(json_obj, dict):
+            # Skip unsupported payloads
+            logger.debug(
+                "sse_serialization_skipped_non_dict",
+                chunk_number=chunk_count,
+                payload_type=type(json_obj).__name__,
+                request_id=request_id,
+                category="sse_format",
+            )
+            continue
+
+        event_type = json_obj.get("type")
+        if isinstance(event_type, str) and event_type:
+            anthropic_chunks += 1
+            if event_type == "ping":
+                sse_event = formatter.format_ping()
+            else:
+                sse_event = formatter.format_event(event_type, json_obj)
+
+            logger.trace(
+                "sse_serialization_anthropic_format",
+                event_type=event_type,
+                chunk_number=chunk_count,
+                request_id=request_id,
+                category="sse_format",
+            )
+        else:
+            openai_chunks += 1
+            json_str = json.dumps(json_obj, ensure_ascii=False)
+            sse_event = f"data: {json_str}\n\n"
+
+            logger.trace(
+                "sse_serialization_openai_format",
+                chunk_number=chunk_count,
+                has_choices=bool(json_obj.get("choices")),
+                request_id=request_id,
+                category="sse_format",
+            )
+
+        yield sse_event.encode("utf-8")
+
+    logger.debug(
+        "sse_serialization_complete",
+        total_chunks=chunk_count,
+        anthropic_chunks=anthropic_chunks,
+        openai_chunks=openai_chunks,
+        request_id=request_id,
+        category="sse_format",
+    )
+
+    if include_done:
+        yield b"data: [DONE]\n\n"
+
+
+__all__ = ["serialize_json_to_sse_stream"]

ccproxy/streaming/sse_parser.py

@@ -0,0 +1,127 @@
+"""Helpers for incrementally parsing server-sent events (SSE)."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+
+class SSEStreamParser:
+    """Accumulate SSE fragments and yield decoded ``data:`` payloads.
+
+    The parser keeps track of partial lines and events across ``feed`` calls so
+    callers can push raw provider chunks (``str`` or ``bytes``) and only receive
+    payloads when a full SSE event has been received. ``data: [DONE]`` sentinel
+    events are filtered out automatically.
+    """
+
+    __slots__ = ("_line_remainder", "_event_lines", "_errors")
+
+    def __init__(self) -> None:
+        self._line_remainder: str = ""
+        self._event_lines: list[str] = []
+        self._errors: list[tuple[str, Exception]] = []
+
+    def feed(self, chunk: str | bytes | None) -> list[Any]:
+        """Process a streaming fragment and return decoded JSON payloads.
+
+        Args:
+            chunk: Raw chunk from the provider. ``bytes`` inputs are decoded
+                using UTF-8. ``None`` or empty values yield no events.
+
+        Returns:
+            List of decoded JSON payloads for completed events. ``[DONE]``
+            sentinels are omitted.
+        """
+
+        if not chunk:
+            return []
+
+        if isinstance(chunk, bytes):
+            chunk = chunk.decode("utf-8", errors="ignore")
+
+        if not chunk:
+            return []
+
+        chunk = chunk.replace("\r\n", "\n").replace("\r", "\n")
+        buffered = f"{self._line_remainder}{chunk}"
+
+        lines = buffered.split("\n")
+        if buffered.endswith("\n"):
+            self._line_remainder = ""
+        else:
+            self._line_remainder = lines.pop()
+
+        completed: list[Any] = []
+
+        for line in lines:
+            if line == "":
+                payload = self._finalize_event()
+                if payload:
+                    completed.append(payload)
+                continue
+
+            self._event_lines.append(line)
+
+        return completed
+
+    def flush(self) -> list[Any]:
+        """Return any buffered payload when the stream ends."""
+
+        if self._line_remainder:
+            self._event_lines.append(self._line_remainder)
+            self._line_remainder = ""
+
+        payload = self._finalize_event()
+        return [payload] if payload else []
+
+    def consume_errors(self) -> list[tuple[str, Exception]]:
+        """Return and clear parsing errors captured since the last call."""
+
+        errors = self._errors
+        self._errors = []
+        return errors
+
+    def _finalize_event(self) -> Any | None:
+        if not self._event_lines:
+            return None
+
+        event_lines, self._event_lines = self._event_lines, []
+
+        data_fields: list[str] = []
+        for line in event_lines:
+            if line.startswith("data:"):
+                data_fields.append(line[5:].lstrip(" "))
+
+        if not data_fields:
+            return None
+
+        # Try newline-preserving join first (Anthropic style), then a collapsed
+        # join for providers that stream JSON without explicit newlines.
+        candidates = ["\n".join(data_fields).strip()]
+        collapsed = "".join(data_fields).strip()
+        if collapsed and collapsed != candidates[0]:
+            candidates.append(collapsed)
+
+        last_exception: json.JSONDecodeError | None = None
+
+        for candidate in candidates:
+            if not candidate or candidate == "[DONE]":
+                return None
+
+            try:
+                return json.loads(candidate)
+            except json.JSONDecodeError as exc:
+                last_exception = exc
+                continue
+
+        if last_exception:
+            # Reconstruct the raw event so we can retry when more data arrives.
+            raw_event = "\n".join(event_lines) + "\n\n"
+            self._line_remainder = f"{raw_event}{self._line_remainder}"
+            self._errors.append((raw_event, last_exception))
+
+        return None
+
+
+__all__ = ["SSEStreamParser"]

ccproxy/templates/__init__.py

@@ -0,0 +1,6 @@
+"""Template helpers for generating CCProxy plugin scaffolds."""
+
+from .plugin_scaffold import PluginTemplateType, build_plugin_scaffold
+
+
+__all__ = ["PluginTemplateType", "build_plugin_scaffold"]