synth-ai 0.2.9.dev0__py3-none-any.whl → 0.2.23.dev3__py3-none-any.whl

examples/warming_up_to_rl/task_app/synth_envs_hosted/inference/openai_client.py

@@ -0,0 +1,729 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+import os
+from typing import Any
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+
+class OpenAIClient:
+    """Async HTTP client for OpenAI-compatible inference servers (vLLM)."""
+
+    def __init__(
+        self,
+        base_url: str,
+        api_key: str | None = None,
+        timeout_s: float = 120.0,
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.api_key = api_key
+        self.timeout_s = timeout_s
+        self.headers = {}
+        self._env_api_key: str | None = None
+
+        try:
+            env_key = os.getenv("ENVIRONMENT_API_KEY") or ""
+            env_key = env_key.strip()
+            if env_key:
+                self._env_api_key = env_key
+        except Exception:
+            self._env_api_key = None
+
+        if api_key:
+            self.headers["Authorization"] = f"Bearer {api_key}"
+
+    def _fix_model_parameters(
+        self, request: dict[str, Any], target_url: str | None = None
+    ) -> dict[str, Any]:
+        """
+        Fix parameter compatibility for newer OpenAI models.
+
+        Newer models like gpt-5-nano use 'max_completion_tokens' instead of 'max_tokens'.
+        """
+        if not request:
+            return request
+
+        # Make a copy to avoid modifying the original
+        fixed_request = request.copy()
+
+        # Determine if target is OpenAI-compatible (OpenAI, Azure OpenAI, Groq);
+        # strip fields those endpoints don't accept
+        is_openai = False
+        try:
+            if isinstance(target_url, str):
+                low = target_url.lower()
+                is_openai = (
+                    ("openai.com" in low)
+                    or ("azure" in low and ".openai." in low)
+                    or ("groq.com" in low)
+                    or ("/openai" in low)
+                )
+        except Exception:
+            is_openai = False
+
+        model = fixed_request.get("model", "")
+
+        if is_openai:
+            # Remove fields OpenAI/Groq don't accept
+            for k in (
+                "stop_after_tool_calls",
+                "thinking_mode",
+                "thinking_budget",
+                "reasoning",
+                "extra_body",
+                "parallel_tool_calls",
+                "function_call",
+            ):
+                if k in fixed_request:
+                    fixed_request.pop(k, None)
+
+            # GPT-5 family specifics
+            if "gpt-5" in model or "gpt-4.1" in model:
+                # Convert max_tokens to max_completion_tokens for newer models
+                if "max_tokens" in fixed_request:
+                    if "max_completion_tokens" not in fixed_request:
+                        fixed_request["max_completion_tokens"] = fixed_request.pop("max_tokens")
+                        logger.info(
+                            f"Converted max_tokens to max_completion_tokens for model {model}"
+                        )
+                    else:
+                        fixed_request.pop("max_tokens")
+                        logger.info(f"Removed conflicting max_tokens parameter for model {model}")
+                # Some OpenAI endpoints ignore/deny sampling fields for reasoning models
+                for k in ("temperature", "top_p"):
+                    if k in fixed_request:
+                        fixed_request.pop(k, None)
+                # If tools are present, force single tool choice to our function
+                try:
+                    tools = fixed_request.get("tools")
+                    if isinstance(tools, list) and tools:
+                        # Choose the first provided function name from tools schema (e.g., run_command)
+                        func_name = None
+                        for t in tools:
+                            try:
+                                cand = None
+                                if isinstance(t, dict):
+                                    f = t.get("function")
+                                    if isinstance(f, dict):
+                                        cand = f.get("name")
+                                if isinstance(cand, str) and cand:
+                                    func_name = cand
+                                    break
+                            except Exception:
+                                continue
+                        if not func_name:
+                            func_name = "run_command"
+                        fixed_request["tool_choice"] = {
+                            "type": "function",
+                            "function": {"name": func_name},
+                        }
+                        fixed_request["parallel_tool_calls"] = False
+                except Exception:
+                    pass
+
+        return fixed_request
+
+    async def generate(
+        self,
+        request: dict[str, Any],
+        base_url: str | None = None,
+        timeout_s: float | None = None,
+        extra_headers: dict[str, str] | None = None,
+    ) -> dict[str, Any]:
+        """
+        Send a chat completion request to the inference server.
+
+        Args:
+            request: OpenAI-compatible chat completion request
+            base_url: Override base URL for this request
+            timeout_s: Override timeout for this request
+            extra_headers: Additional headers to include (e.g., X-Policy-Name)
+
+        Returns:
+            OpenAI-compatible chat completion response
+        """
+        # Build target URL robustly: if a full endpoint is given (with query or already ending
+        # in /chat/completions), preserve it; otherwise, append the path BEFORE query params.
+        from urllib.parse import urlparse, urlunparse
+
+        candidate = (base_url or self.base_url).strip()
+        try:
+            parsed = urlparse(candidate)
+            # If no scheme, treat as relative base (pass-through)
+            if not parsed.scheme or not parsed.netloc:
+                base_no_slash = candidate.rstrip("/")
+                url = f"{base_no_slash}/v1/chat/completions"
+            else:
+                path = (parsed.path or "").rstrip("/")
+                if path.endswith("/v1/chat/completions") or path.endswith("/chat/completions"):
+                    new_path = path
+                elif path.endswith("/v1"):
+                    new_path = f"{path}/chat/completions"
+                elif path.endswith("/chat"):
+                    new_path = f"{path}/completions"
+                else:
+                    new_path = f"{path}/v1/chat/completions" if path else "/v1/chat/completions"
+                url = urlunparse(parsed._replace(path=new_path))
+        except Exception:
+            # Fallback to legacy behavior
+            url = (base_url or self.base_url).rstrip("/") + "/v1/chat/completions"
+        timeout = timeout_s or self.timeout_s
+
+        # Merge headers
+        headers = self.headers.copy()
+        try:
+            parsed_target = urlparse(url)
+            path_for_auth = (parsed_target.path or "") if parsed_target else ""
+            if self._env_api_key and "/proxy/" in path_for_auth:
+                headers.setdefault("X-API-Key", self._env_api_key)
+        except Exception:
+            pass
+        if extra_headers:
+            headers.update(extra_headers)
+
+        # Fix parameter compatibility for newer models
+        processed_request = self._fix_model_parameters(request, target_url=url)
+
+        # Log request with detailed prompts/tools preview and sampling settings (Authorization is not logged)
+        logger.info(f"Inference POST target: {url}")
+        if extra_headers:
+            logger.info(f"Extra headers: {extra_headers}")
+        with contextlib.suppress(Exception):
+            keys_preview = sorted(processed_request.keys())
+            logger.info(f"Request keys: {keys_preview}")
+
+        # Detailed IO log: messages/tools/sampling and final payload fields
+        try:
+            import json as _json
+
+            def _truncate(text: str, limit: int = 2000) -> str:
+                return text if len(text) <= limit else text[:limit] + "…"
+
+            def _messages_preview(msgs: Any) -> str:
+                try:
+                    out: list[dict[str, Any]] = []
+                    if isinstance(msgs, list):
+                        for m in msgs:
+                            if not isinstance(m, dict):
+                                continue
+                            role = m.get("role")
+                            content = m.get("content")
+                            if isinstance(content, str):
+                                text = content
+                            elif isinstance(content, list):
+                                parts: list[str] = []
+                                for seg in content:
+                                    if isinstance(seg, dict) and isinstance(seg.get("text"), str):
+                                        parts.append(seg["text"]) 
+                                text = "\n".join(parts)
+                            else:
+                                text = ""
+                            out.append({"role": role, "content": _truncate(str(text), 4000)})
+                    return _json.dumps(out)
+                except Exception:
+                    return "[]"
+
+            def _tools_preview(tools: Any) -> str:
+                try:
+                    return _truncate(_json.dumps(tools), 4000)
+                except Exception:
+                    return "[]"
+
+            msgs = processed_request.get("messages") if isinstance(processed_request, dict) else None
+            tools = processed_request.get("tools") if isinstance(processed_request, dict) else None
+            io_log: dict[str, Any] = {
+                "llm.call": True,
+                "model": processed_request.get("model") if isinstance(processed_request, dict) else None,
+                "tool_choice": processed_request.get("tool_choice") if isinstance(processed_request, dict) else None,
+                "parallel_tool_calls": processed_request.get("parallel_tool_calls") if isinstance(processed_request, dict) else None,
+                "stop_after_tool_calls": processed_request.get("stop_after_tool_calls") if isinstance(processed_request, dict) else None,
+                "temperature": processed_request.get("temperature") if isinstance(processed_request, dict) else None,
+                "top_p": processed_request.get("top_p") if isinstance(processed_request, dict) else None,
+                "max_tokens": processed_request.get("max_tokens") if isinstance(processed_request, dict) else None,
+                "max_completion_tokens": processed_request.get("max_completion_tokens") if isinstance(processed_request, dict) else None,
+                "messages_preview": _messages_preview(msgs),
+                "tools_preview": _tools_preview(tools),
+            }
+            logger.info(io_log)
+        except Exception:
+            pass
+
+        # Final hard-guard for OpenAI/Groq: ensure unsupported field is not present
+        try:
+            low_url = url.lower()
+            if ("openai" in low_url or "groq.com" in low_url or "/proxy/groq" in low_url) and "stop_after_tool_calls" in processed_request:
+                processed_request.pop("stop_after_tool_calls", None)
+                logger.info("Removed stop_after_tool_calls for Groq/OpenAI request")
+            # Groq-specific requirement: when using JSON mode, one of the messages must contain the word 'json'
+            if ("groq.com" in low_url or "/openai" in low_url) and isinstance(
+                processed_request, dict
+            ):
+                rf = processed_request.get("response_format")
+                rf_type = None
+                if isinstance(rf, dict):
+                    rf_type = str(rf.get("type") or "").lower()
+                if rf_type in {"json_object", "json_schema"}:
+                    msgs = processed_request.get("messages")
+                    has_json_word = False
+                    if isinstance(msgs, list):
+                        for m in msgs:
+                            try:
+                                content = m.get("content") if isinstance(m, dict) else None
+                                text = None
+                                if isinstance(content, str):
+                                    text = content
+                                elif isinstance(content, list):
+                                    # Join any text segments
+                                    parts = []
+                                    for seg in content:
+                                        if isinstance(seg, dict) and isinstance(
+                                            seg.get("text"), str
+                                        ):
+                                            parts.append(seg["text"])
+                                    text = "\n".join(parts)
+                                if isinstance(text, str) and ("json" in text.lower()):
+                                    has_json_word = True
+                                    break
+                            except Exception:
+                                continue
+                    if not has_json_word:
+                        try:
+                            instruction = (
+                                "Respond in strict JSON only. Output a single valid JSON object."
+                            )
+                            if not isinstance(msgs, list):
+                                msgs = []
+                            # Prepend a system message to satisfy Groq requirement without changing user intent
+                            prepend = {"role": "system", "content": instruction}
+                            processed_request["messages"] = [prepend] + list(msgs)
+                            logger.info(
+                                "Injected JSON-mode system instruction for Groq response_format compliance"
+                            )
+                        except Exception:
+                            pass
+        except Exception:
+            pass
+
+        async with httpx.AsyncClient(timeout=timeout) as client:
+            try:
+                response = await client.post(
+                    url,
+                    json=processed_request,
+                    headers=headers,
+                )
+                response.raise_for_status()
+
+                # Rich response diagnostics
+                content_type = response.headers.get("content-type")
+                body_text = response.text
+                logger.info(
+                    f"Inference response status=200, content-type={content_type}, bytes={len(body_text)}"
+                )
+                if body_text:
+                    # Log raw output with generous preview to debug no-tool-call issues
+                    preview_len = min(4000, len(body_text))
+                    logger.info({
+                        "llm.raw_response": True,
+                        "bytes": len(body_text),
+                        "preview": body_text[:preview_len],
+                    })
+
+                result = response.json()
+                logger.info(f"Inference response parsed_type={type(result).__name__}")
+
+                # Normalize tool calls so downstream always sees a function tool call
+                try:
+                    if isinstance(result, dict):
+                        choices = result.get("choices")
+                        if isinstance(choices, list) and choices:
+                            msg = choices[0].get("message")
+                            if isinstance(msg, dict):
+                                # Prefer tool_calls; if missing but function_call is present, synthesize tool_calls
+                                tc = msg.get("tool_calls")
+                                fc = msg.get("function_call")
+                                if (not isinstance(tc, list) or not tc) and isinstance(fc, dict):
+                                    name = fc.get("name") or "interact_many"
+                                    args = fc.get("arguments") or "{}"
+                                    msg["tool_calls"] = [
+                                        {
+                                            "id": "call_norm",
+                                            "type": "function",
+                                            "function": {"name": name, "arguments": args},
+                                        }
+                                    ]
+                                    # Encourage downstream to treat this as a tool call
+                                    if isinstance(choices[0], dict):
+                                        choices[0]["finish_reason"] = "tool_calls"
+                                # Log tool call count for debugging
+                                try:
+                                    tc2 = msg.get("tool_calls")
+                                    count = len(tc2) if isinstance(tc2, list) else 0
+                                    logger.info({
+                                        "llm.tool_calls": True,
+                                        "count": count,
+                                        "finish_reason": choices[0].get("finish_reason") if isinstance(choices[0], dict) else None,
+                                    })
+                                except Exception:
+                                    pass
+                except Exception:
+                    pass
+
+                return result
+
+            except httpx.TimeoutException:
+                logger.error(f"Request to {url} timed out after {timeout}s")
+                raise
+            except httpx.HTTPStatusError as e:
+                status = e.response.status_code if e.response is not None else None
+                text = e.response.text if e.response is not None else str(e)
+                # Log full body for debugging remote failures
+                try:
+                    logger.error(
+                        {
+                            "openai_http_error": True,
+                            "status": status,
+                            "url": url,
+                            "body": text,
+                        }
+                    )
+                except Exception:
+                    logger.error(f"HTTP error from {url}: {status} - {text}")
+                # For 4xx/5xx, print full sanitized request to aid debugging (especially Groq 400s)
+                try:
+                    redacted_headers = dict(headers)
+                    if "Authorization" in redacted_headers:
+                        redacted_headers["Authorization"] = "***REDACTED***"
+                    logger.error(
+                        {
+                            "request_debug": True,
+                            "status": status,
+                            "target": url,
+                            "headers": redacted_headers,
+                            "payload": processed_request,
+                        }
+                    )
+                except Exception:
+                    pass
+                # Special case: token budget exceeded (OpenAI-compatible error schema)
+                try:
+                    if status == 400 and e.response is not None:
+                        data = e.response.json()
+                        detail = data.get("detail") if isinstance(data, dict) else None
+                        err_code = (detail or {}).get("error") if isinstance(detail, dict) else None
+                        if err_code == "token_budget_exceeded":
+                            info = (detail or {}).get("details") or {}
+                            messages_tokens = int(info.get("messages_tokens") or 0)
+                            model_limit = int(info.get("model_limit") or 0)
+                            safety = 64
+                            # Compute a conservative new max_tokens
+                            new_max = max(16, model_limit - messages_tokens - safety)
+                            try:
+                                # Update request and retry once immediately with smaller budget
+                                if isinstance(processed_request, dict):
+                                    processed_request = dict(processed_request)
+                                    if "max_completion_tokens" in processed_request:
+                                        processed_request["max_completion_tokens"] = new_max
+                                        processed_request.pop("max_tokens", None)
+                                    else:
+                                        processed_request["max_tokens"] = new_max
+                                    # Remove optional fields that some servers reject
+                                    for k in ("thinking_mode", "thinking_budget", "reasoning"):
+                                        processed_request.pop(k, None)
+                                    # Force structured tool choice
+                                    if processed_request.get("tool_choice") == "required":
+                                        func_name = "run_command"
+                                        try:
+                                            tools_arr = processed_request.get("tools") or []
+                                            if isinstance(tools_arr, list) and tools_arr:
+                                                f = (
+                                                    tools_arr[0].get("function")
+                                                    if isinstance(tools_arr[0], dict)
+                                                    else None
+                                                )
+                                                cand = (
+                                                    (f or {}).get("name")
+                                                    if isinstance(f, dict)
+                                                    else None
+                                                )
+                                                if isinstance(cand, str) and cand:
+                                                    func_name = cand
+                                        except Exception:
+                                            pass
+                                        processed_request["tool_choice"] = {
+                                            "type": "function",
+                                            "function": {"name": func_name},
+                                        }
+                                        processed_request["parallel_tool_calls"] = False
+                                    logger.warning(
+                                        {
+                                            "token_budget_recovery": True,
+                                            "messages_tokens": messages_tokens,
+                                            "model_limit": model_limit,
+                                            "retry_max_tokens": new_max,
+                                        }
+                                    )
+                                    # Retry once with reduced budget
+                                    async with httpx.AsyncClient(timeout=timeout) as client2:
+                                        r2 = await client2.post(
+                                            url, json=processed_request, headers=headers
+                                        )
+                                        r2.raise_for_status()
+                                        return r2.json()
+                            except Exception:
+                                pass
+                except Exception:
+                    pass
+                raise
+            except Exception as e:
+                logger.error(f"Unexpected error calling {url}: {e}")
+                raise
+
+    async def check_health(
+        self,
+        base_url: str | None = None,
+        timeout_s: float | None = None,
+    ) -> dict[str, Any]:
+        """
+        Check if the inference service is healthy.
+
+        Args:
+            base_url: Override base URL for this request
+            timeout_s: Override timeout for this request
+
+        Returns:
+            Health status dict with 'status' field
+        """
+        url = (base_url or self.base_url).rstrip("/") + "/health"
+        timeout = timeout_s or 10.0
+
+        try:
+            async with httpx.AsyncClient(timeout=timeout) as client:
+                headers = self.headers.copy()
+                try:
+                    parsed = httpx.URL(url)
+                    if self._env_api_key and "/proxy/" in (parsed.path or ""):
+                        headers.setdefault("X-API-Key", self._env_api_key)
+                except Exception:
+                    pass
+                response = await client.get(url, headers=headers)
+                response.raise_for_status()
+                return response.json()
+        except httpx.HTTPStatusError as e:
+            if e.response.status_code == 400:
+                # Service is overloaded but still responding
+                try:
+                    data = e.response.json()
+                    if data.get("status") == "overloaded":
+                        return {"status": "overloaded", "retry_after": data.get("retry_after", 1)}
+                except Exception:
+                    pass
+            return {"status": "unhealthy", "error": str(e)}
+        except Exception as e:
+            return {"status": "unhealthy", "error": str(e)}
+
+    async def generate_with_retries(
+        self,
+        request: dict[str, Any],
+        base_url: str | None = None,
+        timeout_s: float | None = None,
+        max_retries: int = 4,
+        backoff_factor: float = 2.0,
+        extra_headers: dict[str, str] | None = None,
+    ) -> dict[str, Any]:
+        """
+        Generate with exponential backoff retries for transient errors.
+
+        Args:
+            request: OpenAI-compatible chat completion request
+            base_url: Override base URL
+            timeout_s: Override timeout
+            max_retries: Maximum number of retry attempts
+            backoff_factor: Exponential backoff multiplier
+            extra_headers: Additional headers to include (e.g., X-Policy-Name)
+
+        Returns:
+            OpenAI-compatible chat completion response
+        """
+        last_error = None
+        wait_time = 1.0
+
+        for attempt in range(max_retries + 1):
+            try:
+                # Apply parameter fixes to the request
+                processed_request = self._fix_model_parameters(
+                    request,
+                    target_url=(base_url or self.base_url).rstrip("/") + "/v1/chat/completions",
+                )
+                return await self.generate(
+                    request=processed_request,
+                    base_url=base_url,
+                    timeout_s=timeout_s,
+                    extra_headers=extra_headers,
+                )
+            except httpx.HTTPStatusError as e:
+                # Retry on 400 (overloaded), 429 (rate limit), 500 (internal error), 503 (service unavailable)
+                if e.response.status_code not in [400, 429, 500, 503]:
+                    raise
+                last_error = e
+                if e.response.status_code == 400:
+                    # Check if this is an overload error by looking at response content
+                    try:
+                        response_data = e.response.json()
+                        if response_data.get("status") == "overloaded":
+                            retry_after = response_data.get("retry_after", 1)
+                            # Use the suggested retry_after time instead of exponential backoff for overload
+                            wait_time = max(wait_time, float(retry_after))
+                            logger.warning(
+                                f"Inference service overloaded (400). {response_data} Retrying after {wait_time}s..."
+                            )
+                        else:
+                            # This is a different type of 400 error, don't retry
+                            try:
+                                redacted_headers = {}
+                                try:
+                                    redacted_headers = dict(self.headers)
+                                    if "Authorization" in redacted_headers:
+                                        redacted_headers["Authorization"] = "***REDACTED***"
+                                except Exception:
+                                    redacted_headers = {}
+                                logger.error(
+                                    {
+                                        "non_overload_400": True,
+                                        "target": (base_url or self.base_url),
+                                        "payload": processed_request,
+                                        "headers": redacted_headers,
+                                        "body": e.response.text if e.response is not None else None,
+                                    }
+                                )
+                            except Exception:
+                                pass
+                            raise RuntimeError(
+                                f"Inference 400 response: {e.response.text if e.response is not None else 'Bad Request'}"
+                            ) from e
+                    except Exception:
+                        # If we can't parse the response, don't retry 400 errors
+                        with contextlib.suppress(Exception):
+                            logger.error(
+                                {
+                                    "non_overload_400_unparsed": True,
+                                    "target": (base_url or self.base_url),
+                                    "payload": processed_request,
+                                }
+                            )
+                        raise RuntimeError(
+                            f"Inference 400 response (unparsed): {e.response.text if e.response is not None else 'Bad Request'}"
+                        ) from e
+                elif e.response.status_code == 503:
+                    # Avoid referencing undefined response_data
+                    try:
+                        preview = (e.response.text or "")[:200]
+                    except Exception:
+                        preview = ""
+                    logger.warning(
+                        f"Flash returned 503; container may be cold starting. Retrying... body={preview}"
+                    )
+                elif e.response.status_code == 500:
+                    try:
+                        preview = (e.response.text or "")[:200]
+                    except Exception:
+                        preview = ""
+                    logger.warning(
+                        f"Flash returned 500; inference service error. Retrying... body={preview}"
+                    )
+            except httpx.TimeoutException as e:
+                last_error = e
+
+            if attempt < max_retries:
+                logger.warning(
+                    f"Inference request failed (attempt {attempt + 1}/{max_retries + 1}), "
+                    f"retrying in {wait_time}s..."
+                )
+                await asyncio.sleep(wait_time)
+                wait_time *= backoff_factor
+
+        raise last_error
+
+
+def create_inference_client(
+    task_app: Any,
+    api_key: str | None = None,
+) -> OpenAIClient:
+    """
+    Create an inference client using TaskApp configuration.
+
+    Args:
+        task_app: TaskApp instance with vllm_base_url
+        api_key: Optional API key for authentication
+
+    Returns:
+        Configured OpenAIClient instance
+    """
+    # Fallback to environment if caller didn't provide an API key
+    if api_key is None:
+        try:
+            import os as _os  # local import to avoid module-level side effects
+
+            api_key = _os.getenv("OPENAI_API_KEY") or getattr(task_app, "openai_api_key", None)
+        except Exception:
+            api_key = None
+
+    import json as _json
+    import os as _os
+    import time as _time
+
+    if _os.getenv("SYNTH_FAKE_INFERENCE", "").strip():
+
+        class _DummyClient:
+            async def generate_with_retries(
+                self,
+                request: dict[str, Any],
+                base_url: str | None = None,
+                max_retries: int = 0,
+                backoff_factor: float = 1.0,
+                extra_headers: dict[str, str] | None = None,
+            ) -> dict[str, Any]:
+                tool_call = {
+                    "id": "call_dummy",
+                    "type": "function",
+                    "function": {
+                        "name": "interact_many",
+                        "arguments": _json.dumps({"actions": ["move_right"]}),
+                    },
+                }
+                return {
+                    "id": f"cmpl-{int(_time.time())}",
+                    "object": "chat.completion",
+                    "created": int(_time.time()),
+                    "model": request.get("model") or "dummy-model",
+                    "choices": [
+                        {
+                            "index": 0,
+                            "message": {
+                                "role": "assistant",
+                                "content": "",
+                                "tool_calls": [tool_call],
+                            },
+                            "finish_reason": "tool_calls",
+                        }
+                    ],
+                    "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+                }
+
+            async def check_health(
+                self,
+                base_url: str | None = None,
+                timeout_s: float | None = None,
+            ) -> dict[str, Any]:
+                return {"status": "ok", "dummy": True}
+
+        return _DummyClient()
+
+    return OpenAIClient(
+        base_url=task_app.vllm_base_url,
+        api_key=api_key,
+    )