sglang 0.2.10__py3-none-any.whl → 0.2.12__py3-none-any.whl

sglang/srt/models/llama_embedding.py

@@ -0,0 +1,88 @@
+from typing import Iterable, Optional, Tuple
+
+import torch
+from torch import nn
+from transformers import LlamaConfig
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
+
+from sglang.srt.layers.pooler import EmbeddingPoolerOutput, Pooler, PoolingType
+from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.models.llama2 import LlamaForCausalLM, LlamaModel
+
+
+class LlamaEmbeddingModel(nn.Module):
+    def __init__(
+        self,
+        config: LlamaConfig,
+        quant_config=None,
+        cache_config=None,
+        efficient_weight_load=False,
+    ) -> None:
+        super().__init__()
+        self.model = LlamaModel(config, quant_config=quant_config)
+        self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
+
+    @torch.no_grad()
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        positions: torch.Tensor,
+        input_metadata: InputMetadata,
+        input_embeds: torch.Tensor = None,
+    ) -> EmbeddingPoolerOutput:
+        hidden_states = self.model(input_ids, positions, input_metadata, input_embeds)
+        return self.pooler(hidden_states, input_metadata)
+
+    def load_weights(
+        self, weights: Iterable[Tuple[str, torch.Tensor]], name=None, loaded_weight=None
+    ):
+        stacked_params_mapping = [
+            # (param_name, shard_name, shard_id)
+            ("qkv_proj", "q_proj", "q"),
+            ("qkv_proj", "k_proj", "k"),
+            ("qkv_proj", "v_proj", "v"),
+            ("gate_up_proj", "gate_proj", 0),
+            ("gate_up_proj", "up_proj", 1),
+        ]
+        params_dict = dict(self.model.named_parameters())
+
+        def load_weights_per_param(name, loaded_weight):
+            if "rotary_emb.inv_freq" in name or "projector" in name:
+                return
+            if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name:
+                # Models trained using ColossalAI may include these tensors in
+                # the checkpoint. Skip them.
+                return
+            for param_name, weight_name, shard_id in stacked_params_mapping:
+                if weight_name not in name:
+                    continue
+                name = name.replace(weight_name, param_name)
+                # Skip loading extra bias for GPTQ models.
+                if name.endswith(".bias") and name not in params_dict:
+                    continue
+                if name.startswith("model.vision_tower") and name not in params_dict:
+                    continue
+                param = params_dict[name]
+                weight_loader = param.weight_loader
+                weight_loader(param, loaded_weight, shard_id)
+                break
+            else:
+                # Skip loading extra bias for GPTQ models.
+                if name.endswith(".bias") and name not in params_dict:
+                    return
+                if name.startswith("model.vision_tower") and name not in params_dict:
+                    return
+                param = params_dict[name]
+                weight_loader = getattr(param, "weight_loader", default_weight_loader)
+                weight_loader(param, loaded_weight)
+
+        if name is None or loaded_weight is None:
+            for name, loaded_weight in weights:
+                load_weights_per_param(name, loaded_weight)
+        else:
+            load_weights_per_param(name, loaded_weight)
+
+
+EntryClass = LlamaEmbeddingModel
+# compat: e5-mistral model.config class == MistralModel
+EntryClassRemapping = [("MistralModel", LlamaEmbeddingModel)]

sglang/srt/models/llava.py

@@ -32,13 +32,12 @@ from vllm.config import CacheConfig
 from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 
-from sglang.srt.managers.schedule_batch import ForwardMode
 from sglang.srt.mm_utils import (
     get_anyres_image_grid_shape,
     unpad_image,
     unpad_image_shape,
 )
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import ForwardMode, InputMetadata
 from sglang.srt.models.llama2 import LlamaForCausalLM
 from sglang.srt.models.mistral import MistralForCausalLM
 from sglang.srt.models.qwen2 import Qwen2ForCausalLM

sglang/srt/models/llavavid.py

@@ -26,13 +26,12 @@ from vllm.config import CacheConfig
 from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 
-from sglang.srt.managers.schedule_batch import ForwardMode
 from sglang.srt.mm_utils import (
     get_anyres_image_grid_shape,
     unpad_image,
     unpad_image_shape,
 )
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import ForwardMode, InputMetadata
 from sglang.srt.models.llama2 import LlamaForCausalLM
 
 

sglang/srt/models/minicpm.py

@@ -39,7 +39,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 
 from sglang.srt.layers.logits_processor import LogitsProcessor
 from sglang.srt.layers.radix_attention import RadixAttention
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import InputMetadata
 
 
 class MiniCPMMLP(nn.Module):

sglang/srt/models/mixtral.py

@@ -50,7 +50,7 @@ from vllm.utils import print_warning_once
 
 from sglang.srt.layers.logits_processor import LogitsProcessor
 from sglang.srt.layers.radix_attention import RadixAttention
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import InputMetadata
 
 
 class MixtralMoE(nn.Module):

sglang/srt/models/mixtral_quant.py

@@ -45,7 +45,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 
 from sglang.srt.layers.logits_processor import LogitsProcessor
 from sglang.srt.layers.radix_attention import RadixAttention
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import InputMetadata
 
 
 class MixtralMLP(nn.Module):

sglang/srt/models/qwen.py

@@ -39,7 +39,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 
 from sglang.srt.layers.logits_processor import LogitsProcessor
 from sglang.srt.layers.radix_attention import RadixAttention
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import InputMetadata
 
 
 class QWenMLP(nn.Module):

sglang/srt/models/qwen2.py

@@ -39,7 +39,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 
 from sglang.srt.layers.logits_processor import LogitsProcessor
 from sglang.srt.layers.radix_attention import RadixAttention
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import InputMetadata
 
 Qwen2Config = None
 

sglang/srt/models/qwen2_moe.py

@@ -46,12 +46,10 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
     VocabParallelEmbedding,
 )
 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
-from vllm.model_executor.sampling_metadata import SamplingMetadata
-from vllm.sequence import IntermediateTensors, SamplerOutput
 
 from sglang.srt.layers.logits_processor import LogitsProcessor
 from sglang.srt.layers.radix_attention import RadixAttention
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import InputMetadata
 
 
 class Qwen2MoeMLP(nn.Module):
@@ -368,7 +366,6 @@ class Qwen2MoeForCausalLM(nn.Module):
             config.vocab_size, config.hidden_size, quant_config=quant_config
         )
         self.logits_processor = LogitsProcessor(config)
-        self.sampler = Sampler()
 
     @torch.no_grad()
     def forward(
@@ -394,14 +391,6 @@ class Qwen2MoeForCausalLM(nn.Module):
         )
         return logits
 
-    def sample(
-        self,
-        logits: Optional[torch.Tensor],
-        sampling_metadata: SamplingMetadata,
-    ) -> Optional[SamplerOutput]:
-        next_tokens = self.sampler(logits, sampling_metadata)
-        return next_tokens
-
     def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
         stacked_params_mapping = [
             # (param_name, shard_name, shard_id)

sglang/srt/models/stablelm.py

@@ -40,7 +40,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 
 from sglang.srt.layers.logits_processor import LogitsProcessor
 from sglang.srt.layers.radix_attention import RadixAttention
-from sglang.srt.model_executor.model_runner import InputMetadata
+from sglang.srt.model_executor.forward_batch_info import InputMetadata
 
 
 class StablelmMLP(nn.Module):

sglang/srt/openai_api/adapter.py

@@ -34,7 +34,7 @@ from sglang.srt.conversation import (
     generate_chat_conv,
     register_conv_template,
 )
-from sglang.srt.managers.io_struct import GenerateReqInput
+from sglang.srt.managers.io_struct import EmbeddingReqInput, GenerateReqInput
 from sglang.srt.openai_api.protocol import (
     BatchRequest,
     BatchResponse,
@@ -52,7 +52,11 @@ from sglang.srt.openai_api.protocol import (
     CompletionResponseStreamChoice,
     CompletionStreamResponse,
     DeltaMessage,
+    EmbeddingObject,
+    EmbeddingRequest,
+    EmbeddingResponse,
     ErrorResponse,
+    FileDeleteResponse,
     FileRequest,
     FileResponse,
     LogProbs,
@@ -73,7 +77,7 @@ class FileMetadata:
 batch_storage: Dict[str, BatchResponse] = {}
 file_id_request: Dict[str, FileMetadata] = {}
 file_id_response: Dict[str, FileResponse] = {}
-# map file id to file path in SGlang backend
+# map file id to file path in SGLang backend
 file_id_storage: Dict[str, str] = {}
 
 
@@ -81,6 +85,19 @@ file_id_storage: Dict[str, str] = {}
 storage_dir = None
 
 
+def format_finish_reason(finish_reason) -> Optional[str]:
+    if finish_reason.startswith("None"):
+        return None
+    elif finish_reason.startswith("FINISH_MATCHED"):
+        return "stop"
+    elif finish_reason.startswith("FINISH_LENGTH"):
+        return "length"
+    elif finish_reason.startswith("FINISH_ABORT"):
+        return "abort"
+    else:
+        return "unknown"
+
+
 def create_error_response(
     message: str,
     err_type: str = "BadRequestError",
@@ -174,6 +191,20 @@ async def v1_files_create(file: UploadFile, purpose: str, file_storage_pth: str
         return {"error": "Invalid input", "details": e.errors()}
 
 
+async def v1_delete_file(file_id: str):
+    # Retrieve the file job from the in-memory storage
+    file_response = file_id_response.get(file_id)
+    if file_response is None:
+        raise HTTPException(status_code=404, detail="File not found")
+    file_path = file_id_storage.get(file_id)
+    if file_path is None:
+        raise HTTPException(status_code=404, detail="File not found")
+    os.remove(file_path)
+    del file_id_response[file_id]
+    del file_id_storage[file_id]
+    return FileDeleteResponse(id=file_id, deleted=True)
+
+
 async def v1_batches(tokenizer_manager, raw_request: Request):
     try:
         body = await raw_request.json()
@@ -287,6 +318,13 @@ async def process_batch(tokenizer_manager, batch_id: str, batch_request: BatchRe
         retrieve_batch = batch_storage[batch_id]
         retrieve_batch.output_file_id = output_file_id
         file_id_storage[output_file_id] = output_file_path
+        file_id_response[output_file_id] = FileResponse(
+            id=output_file_id,
+            bytes=os.path.getsize(output_file_path),
+            created_at=int(time.time()),
+            filename=f"{output_file_id}.jsonl",
+            purpose="batch_result",
+        )
         # Update batch status to "completed"
         retrieve_batch.status = "completed"
         retrieve_batch.completed_at = int(time.time())
@@ -297,7 +335,7 @@ async def process_batch(tokenizer_manager, batch_id: str, batch_request: BatchRe
         }
 
     except Exception as e:
-        print("error in SGlang:", e)
+        print("error in SGLang:", e)
         # Update batch status to "failed"
         retrieve_batch = batch_storage[batch_id]
         retrieve_batch.status = "failed"
@@ -335,7 +373,6 @@ async def v1_retrieve_file_content(file_id: str):
 
 
 def v1_generate_request(all_requests):
-
     prompts = []
     sampling_params_list = []
     return_logprobs = []
@@ -356,10 +393,13 @@ def v1_generate_request(all_requests):
             {
                 "temperature": request.temperature,
                 "max_new_tokens": request.max_tokens,
+                "min_new_tokens": request.min_tokens,
                 "stop": request.stop,
+                "stop_token_ids": request.stop_token_ids,
                 "top_p": request.top_p,
                 "presence_penalty": request.presence_penalty,
                 "frequency_penalty": request.frequency_penalty,
+                "repetition_penalty": request.repetition_penalty,
                 "regex": request.regex,
                 "n": request.n,
                 "ignore_eos": request.ignore_eos,
@@ -380,7 +420,7 @@ def v1_generate_request(all_requests):
         else:
             prompt_kwargs = {"input_ids": prompt}
     else:
-        if isinstance(prompts[0], str) or isinstance(propmt[0][0], str):
+        if isinstance(prompts[0], str):
             prompt_kwargs = {"text": prompts}
         else:
             prompt_kwargs = {"input_ids": prompts}
@@ -463,14 +503,18 @@ def v1_generate_response(request, ret, tokenizer_manager, to_file=False):
                 "index": 0,
                 "text": text,
                 "logprobs": logprobs,
-                "finish_reason": ret_item["meta_info"]["finish_reason"],
+                "finish_reason": format_finish_reason(
+                    ret_item["meta_info"]["finish_reason"]
+                ),
             }
         else:
             choice_data = CompletionResponseChoice(
                 index=idx,
                 text=text,
                 logprobs=logprobs,
-                finish_reason=ret_item["meta_info"]["finish_reason"],
+                finish_reason=format_finish_reason(
+                    ret_item["meta_info"]["finish_reason"]
+                ),
             )
 
         choices.append(choice_data)
@@ -500,7 +544,9 @@ def v1_generate_response(request, ret, tokenizer_manager, to_file=False):
             responses.append(response)
         return responses
     else:
-        prompt_tokens = sum(item["meta_info"]["prompt_tokens"] for item in ret)
+        prompt_tokens = sum(
+            ret[i]["meta_info"]["prompt_tokens"] for i in range(0, len(ret), request.n)
+        )
         completion_tokens = sum(item["meta_info"]["completion_tokens"] for item in ret)
         response = CompletionResponse(
             id=ret[0]["meta_info"]["id"],
@@ -583,20 +629,34 @@ async def v1_completions(tokenizer_manager, raw_request: Request):
                         index=0,
                         text=delta,
                         logprobs=logprobs,
-                        finish_reason=content["meta_info"]["finish_reason"],
+                        finish_reason=format_finish_reason(
+                            content["meta_info"]["finish_reason"]
+                        ),
                     )
                     chunk = CompletionStreamResponse(
                         id=content["meta_info"]["id"],
                         object="text_completion",
                         choices=[choice_data],
                         model=request.model,
-                        usage=UsageInfo(
-                            prompt_tokens=prompt_tokens,
-                            completion_tokens=completion_tokens,
-                            total_tokens=prompt_tokens + completion_tokens,
-                        ),
                     )
                     yield f"data: {chunk.model_dump_json()}\n\n"
+                if request.stream_options and request.stream_options.include_usage:
+                    usage = UsageInfo(
+                        prompt_tokens=prompt_tokens,
+                        completion_tokens=completion_tokens,
+                        total_tokens=prompt_tokens + completion_tokens,
+                    )
+
+                    final_usage_chunk = CompletionStreamResponse(
+                        id=str(uuid.uuid4().hex),
+                        choices=[],
+                        model=request.model,
+                        usage=usage,
+                    )
+                    final_usage_data = final_usage_chunk.model_dump_json(
+                        exclude_unset=True, exclude_none=True
+                    )
+                    yield f"data: {final_usage_data}\n\n"
             except ValueError as e:
                 error = create_streaming_error_response(str(e))
                 yield f"data: {error}\n\n"
@@ -624,7 +684,6 @@ async def v1_completions(tokenizer_manager, raw_request: Request):
 
 
 def v1_chat_generate_request(all_requests, tokenizer_manager):
-
     input_ids = []
     sampling_params_list = []
     image_data_list = []
@@ -667,10 +726,13 @@ def v1_chat_generate_request(all_requests, tokenizer_manager):
             {
                 "temperature": request.temperature,
                 "max_new_tokens": request.max_tokens,
+                "min_new_tokens": request.min_tokens,
                 "stop": stop,
+                "stop_token_ids": request.stop_token_ids,
                 "top_p": request.top_p,
                 "presence_penalty": request.presence_penalty,
                 "frequency_penalty": request.frequency_penalty,
+                "repetition_penalty": request.repetition_penalty,
                 "regex": request.regex,
                 "n": request.n,
             }
@@ -707,8 +769,6 @@ def v1_chat_generate_request(all_requests, tokenizer_manager):
 
 def v1_chat_generate_response(request, ret, to_file=False):
     choices = []
-    total_prompt_tokens = 0
-    total_completion_tokens = 0
 
     for idx, ret_item in enumerate(ret):
         logprobs = False
@@ -747,8 +807,6 @@ def v1_chat_generate_response(request, ret, to_file=False):
             choice_logprobs = ChoiceLogprobs(content=token_logprobs)
         else:
             choice_logprobs = None
-        prompt_tokens = ret_item["meta_info"]["prompt_tokens"]
-        completion_tokens = ret_item["meta_info"]["completion_tokens"]
 
         if to_file:
             # to make the choice data json serializable
@@ -756,19 +814,22 @@ def v1_chat_generate_response(request, ret, to_file=False):
                 "index": 0,
                 "message": {"role": "assistant", "content": ret_item["text"]},
                 "logprobs": choice_logprobs,
-                "finish_reason": ret_item["meta_info"]["finish_reason"],
+                "finish_reason": format_finish_reason(
+                    ret_item["meta_info"]["finish_reason"]
+                ),
             }
         else:
             choice_data = ChatCompletionResponseChoice(
                 index=idx,
                 message=ChatMessage(role="assistant", content=ret_item["text"]),
                 logprobs=choice_logprobs,
-                finish_reason=ret_item["meta_info"]["finish_reason"],
+                finish_reason=format_finish_reason(
+                    ret_item["meta_info"]["finish_reason"]
+                ),
             )
 
         choices.append(choice_data)
-        total_prompt_tokens += prompt_tokens
-        total_completion_tokens += completion_tokens
+
     if to_file:
         responses = []
 
@@ -795,14 +856,18 @@ def v1_chat_generate_response(request, ret, to_file=False):
             responses.append(response)
         return responses
     else:
+        prompt_tokens = sum(
+            ret[i]["meta_info"]["prompt_tokens"] for i in range(0, len(ret), request.n)
+        )
+        completion_tokens = sum(item["meta_info"]["completion_tokens"] for item in ret)
         response = ChatCompletionResponse(
             id=ret[0]["meta_info"]["id"],
             model=request.model,
             choices=choices,
             usage=UsageInfo(
-                prompt_tokens=total_prompt_tokens,
-                completion_tokens=total_completion_tokens,
-                total_tokens=total_prompt_tokens + total_completion_tokens,
+                prompt_tokens=prompt_tokens,
+                completion_tokens=completion_tokens,
+                total_tokens=prompt_tokens + completion_tokens,
             ),
         )
         return response
@@ -877,18 +942,15 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
                         choice_data = ChatCompletionResponseStreamChoice(
                             index=0,
                             delta=DeltaMessage(role="assistant"),
-                            finish_reason=content["meta_info"]["finish_reason"],
+                            finish_reason=format_finish_reason(
+                                content["meta_info"]["finish_reason"]
+                            ),
                             logprobs=choice_logprobs,
                         )
                         chunk = ChatCompletionStreamResponse(
                             id=content["meta_info"]["id"],
                             choices=[choice_data],
                             model=request.model,
-                            usage=UsageInfo(
-                                prompt_tokens=prompt_tokens,
-                                completion_tokens=completion_tokens,
-                                total_tokens=prompt_tokens + completion_tokens,
-                            ),
                         )
                         yield f"data: {chunk.model_dump_json()}\n\n"
 
@@ -898,20 +960,34 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
                     choice_data = ChatCompletionResponseStreamChoice(
                         index=0,
                         delta=DeltaMessage(content=delta),
-                        finish_reason=content["meta_info"]["finish_reason"],
+                        finish_reason=format_finish_reason(
+                            content["meta_info"]["finish_reason"]
+                        ),
                         logprobs=choice_logprobs,
                     )
                     chunk = ChatCompletionStreamResponse(
                         id=content["meta_info"]["id"],
                         choices=[choice_data],
                         model=request.model,
-                        usage=UsageInfo(
-                            prompt_tokens=prompt_tokens,
-                            completion_tokens=completion_tokens,
-                            total_tokens=prompt_tokens + completion_tokens,
-                        ),
                     )
                     yield f"data: {chunk.model_dump_json()}\n\n"
+                if request.stream_options and request.stream_options.include_usage:
+                    usage = UsageInfo(
+                        prompt_tokens=prompt_tokens,
+                        completion_tokens=completion_tokens,
+                        total_tokens=prompt_tokens + completion_tokens,
+                    )
+
+                    final_usage_chunk = ChatCompletionStreamResponse(
+                        id=str(uuid.uuid4().hex),
+                        choices=[],
+                        model=request.model,
+                        usage=usage,
+                    )
+                    final_usage_data = final_usage_chunk.model_dump_json(
+                        exclude_unset=True, exclude_none=True
+                    )
+                    yield f"data: {final_usage_data}\n\n"
             except ValueError as e:
                 error = create_streaming_error_response(str(e))
                 yield f"data: {error}\n\n"
@@ -930,7 +1006,6 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
         ).__anext__()
     except ValueError as e:
         return create_error_response(str(e))
-
     if not isinstance(ret, list):
         ret = [ret]
 
@@ -939,6 +1014,81 @@ async def v1_chat_completions(tokenizer_manager, raw_request: Request):
     return response
 
 
+def v1_embedding_request(all_requests, tokenizer_manager):
+    prompts = []
+    sampling_params_list = []
+    first_prompt_type = type(all_requests[0].input)
+
+    for request in all_requests:
+        prompt = request.input
+        assert (
+            type(prompt) == first_prompt_type
+        ), "All prompts must be of the same type in file input settings"
+        prompts.append(prompt)
+
+    if len(all_requests) == 1:
+        prompt = prompts[0]
+        if isinstance(prompt, str) or isinstance(prompt[0], str):
+            prompt_kwargs = {"text": prompt}
+        else:
+            prompt_kwargs = {"input_ids": prompt}
+    else:
+        if isinstance(prompts[0], str) or isinstance(propmt[0][0], str):
+            prompt_kwargs = {"text": prompts}
+        else:
+            prompt_kwargs = {"input_ids": prompts}
+
+    adapted_request = EmbeddingReqInput(
+        **prompt_kwargs,
+    )
+
+    if len(all_requests) == 1:
+        return adapted_request, all_requests[0]
+    return adapted_request, all_requests
+
+
+def v1_embedding_response(ret, model_path, to_file=False):
+    embedding_objects = []
+    prompt_tokens = 0
+    for idx, ret_item in enumerate(ret):
+        embedding_objects.append(
+            EmbeddingObject(
+                embedding=ret[idx]["embedding"],
+                index=idx,
+            )
+        )
+        prompt_tokens += ret[idx]["meta_info"]["prompt_tokens"]
+
+    return EmbeddingResponse(
+        data=embedding_objects,
+        model=model_path,
+        usage=UsageInfo(
+            prompt_tokens=prompt_tokens,
+            total_tokens=prompt_tokens,
+        ),
+    )
+
+
+async def v1_embeddings(tokenizer_manager, raw_request: Request):
+    request_json = await raw_request.json()
+    all_requests = [EmbeddingRequest(**request_json)]
+    adapted_request, request = v1_embedding_request(all_requests, tokenizer_manager)
+
+    try:
+        ret = await tokenizer_manager.generate_request(
+            adapted_request, raw_request
+        ).__anext__()
+    except ValueError as e:
+        return create_error_response(str(e))
+
+    if not isinstance(ret, list):
+        ret = [ret]
+
+    response = v1_embedding_response(ret, tokenizer_manager.model_path)
+
+    return response
+
+
 def to_openai_style_logprobs(
     input_token_logprobs=None,
     output_token_logprobs=None,