xinference 0.16.3__py3-none-any.whl → 1.2.1__py3-none-any.whl

xinference/_compat.py

@@ -60,6 +60,10 @@ from openai.types.chat.chat_completion_stream_options_param import (
     ChatCompletionStreamOptionsParam,
 )
 from openai.types.chat.chat_completion_tool_param import ChatCompletionToolParam
+from openai.types.shared_params.response_format_json_object import (
+    ResponseFormatJSONObject,
+)
+from openai.types.shared_params.response_format_text import ResponseFormatText
 
 OpenAIChatCompletionStreamOptionsParam = create_model_from_typeddict(
     ChatCompletionStreamOptionsParam
@@ -68,6 +72,24 @@ OpenAIChatCompletionToolParam = create_model_from_typeddict(ChatCompletionToolPa
 OpenAIChatCompletionNamedToolChoiceParam = create_model_from_typeddict(
     ChatCompletionNamedToolChoiceParam
 )
+from openai._types import Body
+
+
+class JSONSchema(BaseModel):
+    name: str
+    description: Optional[str] = None
+    schema_: Optional[Dict[str, object]] = Field(alias="schema", default=None)
+    strict: Optional[bool] = None
+
+
+class ResponseFormatJSONSchema(BaseModel):
+    json_schema: JSONSchema
+    type: Literal["json_schema"]
+
+
+ResponseFormat = Union[
+    ResponseFormatText, ResponseFormatJSONObject, ResponseFormatJSONSchema
+]
 
 
 class CreateChatCompletionOpenAI(BaseModel):
@@ -84,8 +106,7 @@ class CreateChatCompletionOpenAI(BaseModel):
     n: Optional[int]
     parallel_tool_calls: Optional[bool]
     presence_penalty: Optional[float]
-    # we do not support this
-    # response_format: ResponseFormat
+    response_format: Optional[ResponseFormat]
     seed: Optional[int]
     service_tier: Optional[Literal["auto", "default"]]
     stop: Union[Optional[str], List[str]]
@@ -100,4 +121,5 @@ class CreateChatCompletionOpenAI(BaseModel):
     tools: Optional[Iterable[OpenAIChatCompletionToolParam]]  # type: ignore
     top_logprobs: Optional[int]
     top_p: Optional[float]
+    extra_body: Optional[Body]
     user: Optional[str]

xinference/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-11-07T16:55:36+0800",
+ "date": "2025-01-24T16:52:57+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "85ab86bf1c0967e45fbec995534cd5a0c9a9c439",
- "version": "0.16.3"
+ "full-revisionid": "a57b99b07b40d1082f69a8fc5b968d56bc3636bc",
+ "version": "1.2.1"
 }
 '''  # END VERSION_JSON
 

xinference/api/restful_api.py

@@ -52,10 +52,14 @@ from xoscar.utils import get_next_port
 
 from .._compat import BaseModel, Field
 from .._version import get_versions
-from ..constants import XINFERENCE_DEFAULT_ENDPOINT_PORT, XINFERENCE_DISABLE_METRICS
+from ..constants import (
+    XINFERENCE_DEFAULT_CANCEL_BLOCK_DURATION,
+    XINFERENCE_DEFAULT_ENDPOINT_PORT,
+    XINFERENCE_DISABLE_METRICS,
+)
 from ..core.event import Event, EventCollectorActor, EventType
 from ..core.supervisor import SupervisorActor
-from ..core.utils import json_dumps
+from ..core.utils import CancelMixin, json_dumps
 from ..types import (
     ChatCompletion,
     Completion,
@@ -90,9 +94,9 @@ class CreateCompletionRequest(CreateCompletion):
 
 class CreateEmbeddingRequest(BaseModel):
     model: str
-    input: Union[str, List[str], List[int], List[List[int]]] = Field(
-        description="The input to embed."
-    )
+    input: Union[
+        str, List[str], List[int], List[List[int]], Dict[str, str], List[Dict[str, str]]
+    ] = Field(description="The input to embed.")
     user: Optional[str] = None
 
     class Config:
@@ -111,6 +115,7 @@ class RerankRequest(BaseModel):
     return_documents: Optional[bool] = False
     return_len: Optional[bool] = False
     max_chunks_per_doc: Optional[int] = None
+    kwargs: Optional[str] = None
 
 
 class TextToImageRequest(BaseModel):
@@ -206,7 +211,7 @@ class BuildGradioImageInterfaceRequest(BaseModel):
     model_ability: List[str]
 
 
-class RESTfulAPI:
+class RESTfulAPI(CancelMixin):
     def __init__(
         self,
         supervisor_address: str,
@@ -484,6 +489,16 @@ class RESTfulAPI:
                 else None
             ),
         )
+        self._router.add_api_route(
+            "/v1/convert_ids_to_tokens",
+            self.convert_ids_to_tokens,
+            methods=["POST"],
+            dependencies=(
+                [Security(self._auth_service, scopes=["models:read"])]
+                if self.is_authenticated()
+                else None
+            ),
+        )
         self._router.add_api_route(
             "/v1/rerank",
             self.rerank,
@@ -1199,6 +1214,19 @@ class RESTfulAPI:
     async def get_address(self) -> JSONResponse:
         return JSONResponse(content=self._supervisor_address)
 
+    async def _get_model_last_error(self, replica_model_uid: bytes, e: Exception):
+        if not isinstance(e, xo.ServerClosed):
+            return e
+        try:
+            model_status = await (await self._get_supervisor_ref()).get_model_status(
+                replica_model_uid.decode("utf-8")
+            )
+            if model_status is not None and model_status.last_error:
+                return Exception(model_status.last_error)
+        except Exception as ex:
+            return ex
+        return e
+
     async def create_completion(self, request: Request) -> Response:
         raw_body = await request.json()
         body = CreateCompletionRequest.parse_obj(raw_body)
@@ -1214,6 +1242,9 @@ class RESTfulAPI:
         raw_kwargs = {k: v for k, v in raw_body.items() if k not in exclude}
         kwargs = body.dict(exclude_unset=True, exclude=exclude)
 
+        # guided_decoding params
+        kwargs.update(self.extract_guided_params(raw_body=raw_body))
+
         # TODO: Decide if this default value override is necessary #1061
         if body.max_tokens is None:
             kwargs["max_tokens"] = max_tokens_field.default
@@ -1254,11 +1285,14 @@ class RESTfulAPI:
                     )
                     return
                 except Exception as ex:
+                    ex = await self._get_model_last_error(model.uid, ex)
                     logger.exception("Completion stream got an error: %s", ex)
                     await self._report_error_event(model_uid, str(ex))
                     # https://github.com/openai/openai-python/blob/e0aafc6c1a45334ac889fe3e54957d309c3af93f/src/openai/_streaming.py#L107
                     yield dict(data=json.dumps({"error": str(ex)}))
                     return
+                finally:
+                    await model.decrease_serve_count()
 
             return EventSourceResponse(stream_results())
         else:
@@ -1266,6 +1300,7 @@ class RESTfulAPI:
                 data = await model.generate(body.prompt, kwargs, raw_params=raw_kwargs)
                 return Response(data, media_type="application/json")
             except Exception as e:
+                e = await self._get_model_last_error(model.uid, e)
                 logger.error(e, exc_info=True)
                 await self._report_error_event(model_uid, str(e))
                 self.handle_request_limit_error(e)
@@ -1297,25 +1332,49 @@ class RESTfulAPI:
         try:
             embedding = await model.create_embedding(body.input, **kwargs)
             return Response(embedding, media_type="application/json")
-        except RuntimeError as re:
-            logger.error(re, exc_info=True)
-            await self._report_error_event(model_uid, str(re))
-            self.handle_request_limit_error(re)
-            raise HTTPException(status_code=400, detail=str(re))
+        except Exception as e:
+            e = await self._get_model_last_error(model.uid, e)
+            logger.error(e, exc_info=True)
+            await self._report_error_event(model_uid, str(e))
+            self.handle_request_limit_error(e)
+            raise HTTPException(status_code=500, detail=str(e))
+
+    async def convert_ids_to_tokens(self, request: Request) -> Response:
+        payload = await request.json()
+        body = CreateEmbeddingRequest.parse_obj(payload)
+        model_uid = body.model
+        exclude = {
+            "model",
+            "input",
+            "user",
+        }
+        kwargs = {key: value for key, value in payload.items() if key not in exclude}
+
+        try:
+            model = await (await self._get_supervisor_ref()).get_model(model_uid)
+        except ValueError as ve:
+            logger.error(str(ve), exc_info=True)
+            await self._report_error_event(model_uid, str(ve))
+            raise HTTPException(status_code=400, detail=str(ve))
         except Exception as e:
             logger.error(e, exc_info=True)
             await self._report_error_event(model_uid, str(e))
             raise HTTPException(status_code=500, detail=str(e))
 
+        try:
+            decoded_texts = await model.convert_ids_to_tokens(body.input, **kwargs)
+            return Response(decoded_texts, media_type="application/json")
+        except Exception as e:
+            e = await self._get_model_last_error(model.uid, e)
+            logger.error(e, exc_info=True)
+            await self._report_error_event(model_uid, str(e))
+            self.handle_request_limit_error(e)
+            raise HTTPException(status_code=500, detail=str(e))
+
     async def rerank(self, request: Request) -> Response:
         payload = await request.json()
         body = RerankRequest.parse_obj(payload)
         model_uid = body.model
-        kwargs = {
-            key: value
-            for key, value in payload.items()
-            if key not in RerankRequest.__annotations__.keys()
-        }
 
         try:
             model = await (await self._get_supervisor_ref()).get_model(model_uid)
@@ -1329,6 +1388,10 @@ class RESTfulAPI:
             raise HTTPException(status_code=500, detail=str(e))
 
         try:
+            if body.kwargs is not None:
+                parsed_kwargs = json.loads(body.kwargs)
+            else:
+                parsed_kwargs = {}
             scores = await model.rerank(
                 body.documents,
                 body.query,
@@ -1336,17 +1399,14 @@ class RESTfulAPI:
                 max_chunks_per_doc=body.max_chunks_per_doc,
                 return_documents=body.return_documents,
                 return_len=body.return_len,
-                **kwargs,
+                **parsed_kwargs,
             )
             return Response(scores, media_type="application/json")
-        except RuntimeError as re:
-            logger.error(re, exc_info=True)
-            await self._report_error_event(model_uid, str(re))
-            self.handle_request_limit_error(re)
-            raise HTTPException(status_code=400, detail=str(re))
         except Exception as e:
+            e = await self._get_model_last_error(model.uid, e)
             logger.error(e, exc_info=True)
             await self._report_error_event(model_uid, str(e))
+            self.handle_request_limit_error(e)
             raise HTTPException(status_code=500, detail=str(e))
 
     async def create_transcriptions(
@@ -1391,13 +1451,11 @@ class RESTfulAPI:
                 **parsed_kwargs,
             )
             return Response(content=transcription, media_type="application/json")
-        except RuntimeError as re:
-            logger.error(re, exc_info=True)
-            await self._report_error_event(model_uid, str(re))
-            raise HTTPException(status_code=400, detail=str(re))
         except Exception as e:
+            e = await self._get_model_last_error(model_ref.uid, e)
             logger.error(e, exc_info=True)
             await self._report_error_event(model_uid, str(e))
+            self.handle_request_limit_error(e)
             raise HTTPException(status_code=500, detail=str(e))
 
     async def create_translations(
@@ -1442,13 +1500,11 @@ class RESTfulAPI:
                 **parsed_kwargs,
             )
             return Response(content=translation, media_type="application/json")
-        except RuntimeError as re:
-            logger.error(re, exc_info=True)
-            await self._report_error_event(model_uid, str(re))
-            raise HTTPException(status_code=400, detail=str(re))
         except Exception as e:
+            e = await self._get_model_last_error(model_ref.uid, e)
             logger.error(e, exc_info=True)
             await self._report_error_event(model_uid, str(e))
+            self.handle_request_limit_error(e)
             raise HTTPException(status_code=500, detail=str(e))
 
     async def create_speech(
@@ -1491,19 +1547,24 @@ class RESTfulAPI:
                 **parsed_kwargs,
             )
             if body.stream:
+
+                async def stream_results():
+                    try:
+                        async for item in out:
+                            yield item
+                    finally:
+                        await model.decrease_serve_count()
+
                 return EventSourceResponse(
-                    media_type="application/octet-stream", content=out
+                    media_type="application/octet-stream", content=stream_results()
                 )
             else:
                 return Response(media_type="application/octet-stream", content=out)
-        except RuntimeError as re:
-            logger.error(re, exc_info=True)
-            await self._report_error_event(model_uid, str(re))
-            self.handle_request_limit_error(re)
-            raise HTTPException(status_code=400, detail=str(re))
         except Exception as e:
+            e = await self._get_model_last_error(model.uid, e)
             logger.error(e, exc_info=True)
             await self._report_error_event(model_uid, str(e))
+            self.handle_request_limit_error(e)
             raise HTTPException(status_code=500, detail=str(e))
 
     async def get_progress(self, request_id: str) -> JSONResponse:
@@ -1531,8 +1592,11 @@ class RESTfulAPI:
             await self._report_error_event(model_uid, str(e))
             raise HTTPException(status_code=500, detail=str(e))
 
+        request_id = None
         try:
             kwargs = json.loads(body.kwargs) if body.kwargs else {}
+            request_id = kwargs.get("request_id")
+            self._add_running_task(request_id)
             image_list = await model.text_to_image(
                 prompt=body.prompt,
                 n=body.n,
@@ -1541,14 +1605,16 @@ class RESTfulAPI:
                 **kwargs,
             )
             return Response(content=image_list, media_type="application/json")
-        except RuntimeError as re:
-            logger.error(re, exc_info=True)
-            await self._report_error_event(model_uid, str(re))
-            self.handle_request_limit_error(re)
-            raise HTTPException(status_code=400, detail=str(re))
+        except asyncio.CancelledError:
+            err_str = f"The request has been cancelled: {request_id}"
+            logger.error(err_str)
+            await self._report_error_event(model_uid, err_str)
+            raise HTTPException(status_code=409, detail=err_str)
         except Exception as e:
+            e = await self._get_model_last_error(model.uid, e)
             logger.error(e, exc_info=True)
             await self._report_error_event(model_uid, str(e))
+            self.handle_request_limit_error(e)
             raise HTTPException(status_code=500, detail=str(e))
 
     async def sdapi_options(self, request: Request) -> Response:
@@ -1619,14 +1685,11 @@ class RESTfulAPI:
                 **kwargs,
             )
             return Response(content=image_list, media_type="application/json")
-        except RuntimeError as re:
-            logger.error(re, exc_info=True)
-            await self._report_error_event(model_uid, str(re))
-            self.handle_request_limit_error(re)
-            raise HTTPException(status_code=400, detail=str(re))
         except Exception as e:
+            e = await self._get_model_last_error(model.uid, e)
             logger.error(e, exc_info=True)
             await self._report_error_event(model_uid, str(e))
+            self.handle_request_limit_error(e)
             raise HTTPException(status_code=500, detail=str(e))
 
     async def sdapi_img2img(self, request: Request) -> Response:
@@ -1653,14 +1716,11 @@ class RESTfulAPI:
                 **kwargs,
             )
             return Response(content=image_list, media_type="application/json")
-        except RuntimeError as re:
-            logger.error(re, exc_info=True)
-            await self._report_error_event(model_uid, str(re))
-            self.handle_request_limit_error(re)
-            raise HTTPException(status_code=400, detail=str(re))
         except Exception as e:
+            e = await self._get_model_last_error(model.uid, e)
             logger.error(e, exc_info=True)
             await self._report_error_event(model_uid, str(e))
+            self.handle_request_limit_error(e)
             raise HTTPException(status_code=500, detail=str(e))
 
     async def create_variations(
@@ -1686,11 +1746,14 @@ class RESTfulAPI:
             await self._report_error_event(model_uid, str(e))
             raise HTTPException(status_code=500, detail=str(e))
 
+        request_id = None
         try:
             if kwargs is not None:
                 parsed_kwargs = json.loads(kwargs)
             else:
                 parsed_kwargs = {}
+            request_id = parsed_kwargs.get("request_id")
+            self._add_running_task(request_id)
             image_list = await model_ref.image_to_image(
                 image=Image.open(image.file),
                 prompt=prompt,
@@ -1701,13 +1764,16 @@ class RESTfulAPI:
                 **parsed_kwargs,
             )
             return Response(content=image_list, media_type="application/json")
-        except RuntimeError as re:
-            logger.error(re, exc_info=True)
-            await self._report_error_event(model_uid, str(re))
-            raise HTTPException(status_code=400, detail=str(re))
+        except asyncio.CancelledError:
+            err_str = f"The request has been cancelled: {request_id}"
+            logger.error(err_str)
+            await self._report_error_event(model_uid, err_str)
+            raise HTTPException(status_code=409, detail=err_str)
         except Exception as e:
+            e = await self._get_model_last_error(model_ref.uid, e)
             logger.error(e, exc_info=True)
             await self._report_error_event(model_uid, str(e))
+            self.handle_request_limit_error(e)
             raise HTTPException(status_code=500, detail=str(e))
 
     async def create_inpainting(
@@ -1734,11 +1800,14 @@ class RESTfulAPI:
             await self._report_error_event(model_uid, str(e))
             raise HTTPException(status_code=500, detail=str(e))
 
+        request_id = None
         try:
             if kwargs is not None:
                 parsed_kwargs = json.loads(kwargs)
             else:
                 parsed_kwargs = {}
+            request_id = parsed_kwargs.get("request_id")
+            self._add_running_task(request_id)
             im = Image.open(image.file)
             mask_im = Image.open(mask_image.file)
             if not size:
@@ -1755,13 +1824,16 @@ class RESTfulAPI:
                 **parsed_kwargs,
             )
             return Response(content=image_list, media_type="application/json")
-        except RuntimeError as re:
-            logger.error(re, exc_info=True)
-            await self._report_error_event(model_uid, str(re))
-            raise HTTPException(status_code=400, detail=str(re))
+        except asyncio.CancelledError:
+            err_str = f"The request has been cancelled: {request_id}"
+            logger.error(err_str)
+            await self._report_error_event(model_uid, err_str)
+            raise HTTPException(status_code=409, detail=err_str)
         except Exception as e:
+            e = await self._get_model_last_error(model_ref.uid, e)
             logger.error(e, exc_info=True)
             await self._report_error_event(model_uid, str(e))
+            self.handle_request_limit_error(e)
             raise HTTPException(status_code=500, detail=str(e))
 
     async def create_ocr(
@@ -1782,24 +1854,30 @@ class RESTfulAPI:
             await self._report_error_event(model_uid, str(e))
             raise HTTPException(status_code=500, detail=str(e))
 
+        request_id = None
         try:
             if kwargs is not None:
                 parsed_kwargs = json.loads(kwargs)
             else:
                 parsed_kwargs = {}
+            request_id = parsed_kwargs.get("request_id")
+            self._add_running_task(request_id)
             im = Image.open(image.file)
             text = await model_ref.ocr(
                 image=im,
                 **parsed_kwargs,
             )
             return Response(content=text, media_type="text/plain")
-        except RuntimeError as re:
-            logger.error(re, exc_info=True)
-            await self._report_error_event(model_uid, str(re))
-            raise HTTPException(status_code=400, detail=str(re))
+        except asyncio.CancelledError:
+            err_str = f"The request has been cancelled: {request_id}"
+            logger.error(err_str)
+            await self._report_error_event(model_uid, err_str)
+            raise HTTPException(status_code=409, detail=err_str)
         except Exception as e:
+            e = await self._get_model_last_error(model_ref.uid, e)
             logger.error(e, exc_info=True)
             await self._report_error_event(model_uid, str(e))
+            self.handle_request_limit_error(e)
             raise HTTPException(status_code=500, detail=str(e))
 
     async def create_flexible_infer(self, request: Request) -> Response:
@@ -1826,14 +1904,11 @@ class RESTfulAPI:
         try:
             result = await model.infer(**kwargs)
             return Response(result, media_type="application/json")
-        except RuntimeError as re:
-            logger.error(re, exc_info=True)
-            await self._report_error_event(model_uid, str(re))
-            self.handle_request_limit_error(re)
-            raise HTTPException(status_code=400, detail=str(re))
         except Exception as e:
+            e = await self._get_model_last_error(model.uid, e)
             logger.error(e, exc_info=True)
             await self._report_error_event(model_uid, str(e))
+            self.handle_request_limit_error(e)
             raise HTTPException(status_code=500, detail=str(e))
 
     async def create_videos(self, request: Request) -> Response:
@@ -1858,14 +1933,11 @@ class RESTfulAPI:
                 **kwargs,
             )
             return Response(content=video_list, media_type="application/json")
-        except RuntimeError as re:
-            logger.error(re, exc_info=True)
-            await self._report_error_event(model_uid, str(re))
-            self.handle_request_limit_error(re)
-            raise HTTPException(status_code=400, detail=str(re))
         except Exception as e:
+            e = await self._get_model_last_error(model.uid, e)
             logger.error(e, exc_info=True)
             await self._report_error_event(model_uid, str(e))
+            self.handle_request_limit_error(e)
             raise HTTPException(status_code=500, detail=str(e))
 
     async def create_chat_completion(self, request: Request) -> Response:
@@ -1880,9 +1952,13 @@ class RESTfulAPI:
             "logit_bias_type",
             "user",
         }
+
         raw_kwargs = {k: v for k, v in raw_body.items() if k not in exclude}
         kwargs = body.dict(exclude_unset=True, exclude=exclude)
 
+        # guided_decoding params
+        kwargs.update(self.extract_guided_params(raw_body=raw_body))
+
         # TODO: Decide if this default value override is necessary #1061
         if body.max_tokens is None:
             kwargs["max_tokens"] = max_tokens_field.default
@@ -1946,7 +2022,6 @@ class RESTfulAPI:
                 )
         if body.tools and body.stream:
             is_vllm = await model.is_vllm_backend()
-
             if not (
                 (is_vllm and model_family in QWEN_TOOL_CALL_FAMILY)
                 or (not is_vllm and model_family in GLM4_TOOL_CALL_FAMILY)
@@ -1956,7 +2031,8 @@ class RESTfulAPI:
                     detail="Streaming support for tool calls is available only when using "
                     "Qwen models with vLLM backend or GLM4-chat models without vLLM backend.",
                 )
-
+        if "skip_special_tokens" in raw_kwargs and await model.is_vllm_backend():
+            kwargs["skip_special_tokens"] = raw_kwargs["skip_special_tokens"]
         if body.stream:
 
             async def stream_results():
@@ -1986,11 +2062,14 @@ class RESTfulAPI:
                     # TODO: Cannot yield here. Yield here would leads to error for the next streaming request.
                     return
                 except Exception as ex:
+                    ex = await self._get_model_last_error(model.uid, ex)
                     logger.exception("Chat completion stream got an error: %s", ex)
                     await self._report_error_event(model_uid, str(ex))
                     # https://github.com/openai/openai-python/blob/e0aafc6c1a45334ac889fe3e54957d309c3af93f/src/openai/_streaming.py#L107
                     yield dict(data=json.dumps({"error": str(ex)}))
                     return
+                finally:
+                    await model.decrease_serve_count()
 
             return EventSourceResponse(stream_results())
         else:
@@ -2002,6 +2081,7 @@ class RESTfulAPI:
                 )
                 return Response(content=data, media_type="application/json")
             except Exception as e:
+                e = await self._get_model_last_error(model.uid, e)
                 logger.error(e, exc_info=True)
                 await self._report_error_event(model_uid, str(e))
                 self.handle_request_limit_error(e)
@@ -2111,10 +2191,25 @@ class RESTfulAPI:
             logger.error(e, exc_info=True)
             raise HTTPException(status_code=500, detail=str(e))
 
-    async def abort_request(self, model_uid: str, request_id: str) -> JSONResponse:
+    async def abort_request(
+        self, request: Request, model_uid: str, request_id: str
+    ) -> JSONResponse:
         try:
+            payload = await request.json()
+            block_duration = payload.get(
+                "block_duration", XINFERENCE_DEFAULT_CANCEL_BLOCK_DURATION
+            )
+            logger.info(
+                "Abort request with model uid: %s, request id: %s, block duration: %s",
+                model_uid,
+                request_id,
+                block_duration,
+            )
             supervisor_ref = await self._get_supervisor_ref()
-            res = await supervisor_ref.abort_request(model_uid, request_id)
+            res = await supervisor_ref.abort_request(
+                model_uid, request_id, block_duration
+            )
+            self._cancel_running_task(request_id, block_duration)
             return JSONResponse(content=res)
         except Exception as e:
             logger.error(e, exc_info=True)
@@ -2228,6 +2323,53 @@ class RESTfulAPI:
             logger.error(e, exc_info=True)
             raise HTTPException(status_code=500, detail=str(e))
 
+    @staticmethod
+    def extract_guided_params(raw_body: dict) -> dict:
+        kwargs = {}
+        raw_extra_body: dict = raw_body.get("extra_body")  # type: ignore
+        if raw_body.get("guided_json"):
+            kwargs["guided_json"] = raw_body.get("guided_json")
+        if raw_body.get("guided_regex") is not None:
+            kwargs["guided_regex"] = raw_body.get("guided_regex")
+        if raw_body.get("guided_choice") is not None:
+            kwargs["guided_choice"] = raw_body.get("guided_choice")
+        if raw_body.get("guided_grammar") is not None:
+            kwargs["guided_grammar"] = raw_body.get("guided_grammar")
+        if raw_body.get("guided_json_object") is not None:
+            kwargs["guided_json_object"] = raw_body.get("guided_json_object")
+        if raw_body.get("guided_decoding_backend") is not None:
+            kwargs["guided_decoding_backend"] = raw_body.get("guided_decoding_backend")
+        if raw_body.get("guided_whitespace_pattern") is not None:
+            kwargs["guided_whitespace_pattern"] = raw_body.get(
+                "guided_whitespace_pattern"
+            )
+        # Parse OpenAI extra_body
+        if raw_extra_body is not None:
+            if raw_extra_body.get("guided_json"):
+                kwargs["guided_json"] = raw_extra_body.get("guided_json")
+            if raw_extra_body.get("guided_regex") is not None:
+                kwargs["guided_regex"] = raw_extra_body.get("guided_regex")
+            if raw_extra_body.get("guided_choice") is not None:
+                kwargs["guided_choice"] = raw_extra_body.get("guided_choice")
+            if raw_extra_body.get("guided_grammar") is not None:
+                kwargs["guided_grammar"] = raw_extra_body.get("guided_grammar")
+            if raw_extra_body.get("guided_json_object") is not None:
+                kwargs["guided_json_object"] = raw_extra_body.get("guided_json_object")
+            if raw_extra_body.get("guided_decoding_backend") is not None:
+                kwargs["guided_decoding_backend"] = raw_extra_body.get(
+                    "guided_decoding_backend"
+                )
+            if raw_extra_body.get("guided_whitespace_pattern") is not None:
+                kwargs["guided_whitespace_pattern"] = raw_extra_body.get(
+                    "guided_whitespace_pattern"
+                )
+            if raw_extra_body.get("platform") is not None:
+                kwargs["platform"] = raw_extra_body.get("platform")
+            if raw_extra_body.get("format") is not None:
+                kwargs["format"] = raw_extra_body.get("format")
+
+        return kwargs
+
 
 def run(
     supervisor_address: str,