sglang 0.4.0.post1__py3-none-any.whl → 0.4.1__py3-none-any.whl

sglang/bench_offline_throughput.py

@@ -201,18 +201,17 @@ def throughput_test_once(
         for r in reqs
     ]
 
-    st = time.perf_counter()
     if profile:
         backend.start_profile()
 
+    st = time.perf_counter()
     gen_out = backend.generate(prompt=prompt, sampling_params=sampling_params)
+    latency = time.perf_counter() - st
 
     if profile:
         backend.stop_profile()
         monitor_trace_file(os.getenv("SGLANG_TORCH_PROFILER_DIR"))
 
-    latency = time.perf_counter() - st
-
     if backend_name == "runtime":
         gen_out = json.loads(gen_out)
 
@@ -285,7 +284,7 @@ def throughput_test(
     else:
         raise ValueError('Please set backend to either "engine" or "runtime"')
 
-    tokenizer_id = server_args.model_path
+    tokenizer_id = server_args.tokenizer_path or server_args.model_path
     tokenizer = get_tokenizer(tokenizer_id)
 
     # Set global environmnets
@@ -304,8 +303,8 @@ def throughput_test(
     warmup_requests = sample_random_requests(
         input_len=256,
         output_len=16,
-        num_prompts=16,
-        range_ratio=0.8,
+        num_prompts=min(bench_args.num_prompts, 16),
+        range_ratio=1.0,
         tokenizer=tokenizer,
         dataset_path=bench_args.dataset_path,
     )
@@ -321,6 +320,7 @@ def throughput_test(
             extra_request_body=extra_request_body,
             profile=False,
         )
+        time.sleep(0.5)
 
     logging.info("\nBenchmark...")
     result = throughput_test_once(

sglang/bench_one_batch.py

@@ -385,6 +385,7 @@ def latency_test(
         8,  # shorter decoding to speed up the warmup
         server_args.device,
     )
+
     rank_print("Benchmark ...")
 
     # Run the sweep

sglang/bench_serving.py

@@ -321,6 +321,8 @@ async def async_request_sglang_generate(
             },
             "stream": not args.disable_stream,
             "lora_path": request_func_input.lora_name,
+            "return_logprob": args.return_logprob,
+            "logprob_start_len": -1,
             **request_func_input.extra_request_body,
         }
         headers = {}
@@ -911,7 +913,7 @@ async def benchmark(
         prompt=test_prompt,
         api_url=api_url,
         prompt_len=test_prompt_len,
-        output_len=test_output_len,
+        output_len=min(test_output_len, 32),
         lora_name=lora_name,
         extra_request_body=extra_request_body,
     )
@@ -922,6 +924,7 @@ async def benchmark(
             f"are correctly specified. Error: {test_output.error}"
         )
     else:
+        requests.post(base_url + "/flush_cache")
         print("Initial test run completed. Starting main benchmark run...")
 
     time.sleep(1.5)
@@ -1413,6 +1416,11 @@ if __name__ == "__main__":
         action="store_true",
         help="Disable ignoring EOS.",
     )
+    parser.add_argument(
+        "--return-logprob",
+        action="store_true",
+        help="Return logprob.",
+    )
     parser.add_argument(
         "--extra-request-body",
         metavar='{"key1": "value1", "key2": "value2"}',

sglang/check_env.py

@@ -9,6 +9,13 @@ from collections import OrderedDict, defaultdict
 
 import torch
 
+from sglang.srt.utils import is_hip
+
+
+def is_cuda_v2():
+    return torch.version.cuda is not None
+
+
 # List of packages to check versions
 PACKAGE_LIST = [
     "sglang",
@@ -63,13 +70,22 @@ def get_cuda_info():
     """
     Get CUDA-related information if available.
     """
-    cuda_info = {"CUDA available": torch.cuda.is_available()}
+    if is_cuda_v2():
+        cuda_info = {"CUDA available": torch.cuda.is_available()}
+
+        if cuda_info["CUDA available"]:
+            cuda_info.update(_get_gpu_info())
+            cuda_info.update(_get_cuda_version_info())
+
+        return cuda_info
+    elif is_hip():
+        cuda_info = {"ROCM available": torch.cuda.is_available()}
 
-    if cuda_info["CUDA available"]:
-        cuda_info.update(_get_gpu_info())
-        cuda_info.update(_get_cuda_version_info())
+        if cuda_info["ROCM available"]:
+            cuda_info.update(_get_gpu_info())
+            cuda_info.update(_get_cuda_version_info())
 
-    return cuda_info
+        return cuda_info
 
 
 def _get_gpu_info():
@@ -103,34 +119,72 @@ def _get_cuda_version_info():
     """
     Get CUDA version information.
     """
-    from torch.utils.cpp_extension import CUDA_HOME
+    if is_cuda_v2():
+        from torch.utils.cpp_extension import CUDA_HOME
 
-    cuda_info = {"CUDA_HOME": CUDA_HOME}
+        cuda_info = {"CUDA_HOME": CUDA_HOME}
 
-    if CUDA_HOME and os.path.isdir(CUDA_HOME):
-        cuda_info.update(_get_nvcc_info())
-        cuda_info.update(_get_cuda_driver_version())
+        if CUDA_HOME and os.path.isdir(CUDA_HOME):
+            cuda_info.update(_get_nvcc_info())
+            cuda_info.update(_get_cuda_driver_version())
 
-    return cuda_info
+        return cuda_info
+    elif is_hip():
+        from torch.utils.cpp_extension import ROCM_HOME as ROCM_HOME
+
+        cuda_info = {"ROCM_HOME": ROCM_HOME}
+
+        if ROCM_HOME and os.path.isdir(ROCM_HOME):
+            cuda_info.update(_get_nvcc_info())
+            cuda_info.update(_get_cuda_driver_version())
+
+        return cuda_info
+    else:
+        cuda_info = {"CUDA_HOME": ""}
+        return cuda_info
 
 
 def _get_nvcc_info():
     """
     Get NVCC version information.
     """
-    from torch.utils.cpp_extension import CUDA_HOME
+    if is_cuda_v2():
+        from torch.utils.cpp_extension import CUDA_HOME
 
-    try:
-        nvcc = os.path.join(CUDA_HOME, "bin/nvcc")
-        nvcc_output = (
-            subprocess.check_output(f'"{nvcc}" -V', shell=True).decode("utf-8").strip()
-        )
-        return {
-            "NVCC": nvcc_output[
-                nvcc_output.rfind("Cuda compilation tools") : nvcc_output.rfind("Build")
-            ].strip()
-        }
-    except subprocess.SubprocessError:
+        try:
+            nvcc = os.path.join(CUDA_HOME, "bin/nvcc")
+            nvcc_output = (
+                subprocess.check_output(f'"{nvcc}" -V', shell=True)
+                .decode("utf-8")
+                .strip()
+            )
+            return {
+                "NVCC": nvcc_output[
+                    nvcc_output.rfind("Cuda compilation tools") : nvcc_output.rfind(
+                        "Build"
+                    )
+                ].strip()
+            }
+        except subprocess.SubprocessError:
+            return {"NVCC": "Not Available"}
+    elif is_hip():
+        from torch.utils.cpp_extension import ROCM_HOME
+
+        try:
+            hipcc = os.path.join(ROCM_HOME, "bin/hipcc")
+            hipcc_output = (
+                subprocess.check_output(f'"{hipcc}" --version', shell=True)
+                .decode("utf-8")
+                .strip()
+            )
+            return {
+                "HIPCC": hipcc_output[
+                    hipcc_output.rfind("HIP version") : hipcc_output.rfind("AMD clang")
+                ].strip()
+            }
+        except subprocess.SubprocessError:
+            return {"HIPCC": "Not Available"}
+    else:
         return {"NVCC": "Not Available"}
 
 
@@ -139,20 +193,40 @@ def _get_cuda_driver_version():
     Get CUDA driver version.
     """
     versions = set()
-    try:
-        output = subprocess.check_output(
-            [
-                "nvidia-smi",
-                "--query-gpu=driver_version",
-                "--format=csv,noheader,nounits",
-            ]
-        )
-        versions = set(output.decode().strip().split("\n"))
-        if len(versions) == 1:
-            return {"CUDA Driver Version": versions.pop()}
-        else:
-            return {"CUDA Driver Versions": ", ".join(sorted(versions))}
-    except subprocess.SubprocessError:
+    if is_cuda_v2():
+        try:
+            output = subprocess.check_output(
+                [
+                    "nvidia-smi",
+                    "--query-gpu=driver_version",
+                    "--format=csv,noheader,nounits",
+                ]
+            )
+            versions = set(output.decode().strip().split("\n"))
+            if len(versions) == 1:
+                return {"CUDA Driver Version": versions.pop()}
+            else:
+                return {"CUDA Driver Versions": ", ".join(sorted(versions))}
+        except subprocess.SubprocessError:
+            return {"CUDA Driver Version": "Not Available"}
+    elif is_hip():
+        try:
+            output = subprocess.check_output(
+                [
+                    "rocm-smi",
+                    "--showdriverversion",
+                    "--csv",
+                ]
+            )
+            versions = set(output.decode().strip().split("\n"))
+            versions.discard("name, value")
+            ver = versions.pop()
+            ver = ver.replace('"Driver version", ', "").replace('"', "")
+
+            return {"ROCM Driver Version": ver}
+        except subprocess.SubprocessError:
+            return {"ROCM Driver Version": "Not Available"}
+    else:
         return {"CUDA Driver Version": "Not Available"}
 
 
@@ -160,16 +234,31 @@ def get_gpu_topology():
     """
     Get GPU topology information.
     """
-    try:
-        result = subprocess.run(
-            ["nvidia-smi", "topo", "-m"],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            text=True,
-            check=True,
-        )
-        return "\n" + result.stdout if result.returncode == 0 else None
-    except subprocess.SubprocessError:
+    if is_cuda_v2():
+        try:
+            result = subprocess.run(
+                ["nvidia-smi", "topo", "-m"],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                check=True,
+            )
+            return "\n" + result.stdout if result.returncode == 0 else None
+        except subprocess.SubprocessError:
+            return None
+    elif is_hip():
+        try:
+            result = subprocess.run(
+                ["rocm-smi", "--showtopotype"],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                check=True,
+            )
+            return "\n" + result.stdout if result.returncode == 0 else None
+        except subprocess.SubprocessError:
+            return None
+    else:
         return None
 
 
@@ -196,7 +285,10 @@ def check_env():
 
     gpu_topo = get_gpu_topology()
     if gpu_topo:
-        env_info["NVIDIA Topology"] = gpu_topo
+        if is_cuda_v2():
+            env_info["NVIDIA Topology"] = gpu_topo
+        elif is_hip():
+            env_info["AMD Topology"] = gpu_topo
 
     hypervisor_vendor = get_hypervisor_vendor()
     if hypervisor_vendor:

sglang/lang/backend/runtime_endpoint.py

@@ -55,6 +55,7 @@ class RuntimeEndpoint(BaseBackend):
             self.base_url + "/flush_cache",
             api_key=self.api_key,
             verify=self.verify,
+            method="POST",
         )
         self._assert_success(res)
 

sglang/lang/chat_template.py

@@ -320,6 +320,28 @@ register_chat_template(
     )
 )
 
+register_chat_template(
+    ChatTemplate(
+        name="granite-3-instruct",
+        default_system_prompt=None,
+        role_prefix_and_suffix={
+            "system": (
+                "<|start_of_role|>system<|end_of_role|>",
+                "<|end_of_text|>",
+            ),
+            "user": (
+                "<|start_of_role|>user<|end_of_role|>",
+                "<|end_of_text|>",
+            ),
+            "assistant": (
+                "<|start_of_role|>assistant<|end_of_role|>",
+                "<|end_of_text|>",
+            ),
+        },
+        stop_str=("<|end_of_text|>",),
+    )
+)
+
 
 @register_chat_template_matching_function
 def match_dbrx(model_path: str):
@@ -402,6 +424,16 @@ def match_c4ai_command_r(model_path: str):
         return get_chat_template("c4ai-command-r")
 
 
+@register_chat_template_matching_function
+def match_granite_instruct(model_path: str):
+    model_path = model_path.lower()
+    # When future versions of Granite are released, this code may
+    # need to be updated. For now, assume that the Granite 3.0
+    # template works across the board.
+    if "granite" in model_path and "instruct" in model_path:
+        return get_chat_template("granite-3-instruct")
+
+
 if __name__ == "__main__":
     messages = [
         {"role": "system", "content": None},  # None means default