sglang 0.2.13__py3-none-any.whl → 0.2.14.post1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sglang/api.py +6 -0
- sglang/bench_latency.py +7 -3
- sglang/bench_serving.py +50 -26
- sglang/check_env.py +15 -0
- sglang/lang/chat_template.py +10 -5
- sglang/lang/compiler.py +4 -0
- sglang/lang/interpreter.py +1 -0
- sglang/lang/ir.py +9 -0
- sglang/launch_server.py +8 -1
- sglang/srt/constrained/fsm_cache.py +11 -2
- sglang/srt/constrained/jump_forward.py +1 -0
- sglang/srt/conversation.py +50 -1
- sglang/srt/hf_transformers_utils.py +22 -23
- sglang/srt/layers/activation.py +100 -1
- sglang/srt/layers/decode_attention.py +338 -50
- sglang/srt/layers/fused_moe/layer.py +2 -2
- sglang/srt/layers/logits_processor.py +56 -19
- sglang/srt/layers/radix_attention.py +3 -4
- sglang/srt/layers/sampler.py +101 -0
- sglang/srt/managers/controller_multi.py +2 -8
- sglang/srt/managers/controller_single.py +7 -10
- sglang/srt/managers/detokenizer_manager.py +20 -9
- sglang/srt/managers/io_struct.py +44 -11
- sglang/srt/managers/policy_scheduler.py +5 -2
- sglang/srt/managers/schedule_batch.py +46 -166
- sglang/srt/managers/tokenizer_manager.py +192 -83
- sglang/srt/managers/tp_worker.py +118 -24
- sglang/srt/mem_cache/memory_pool.py +82 -8
- sglang/srt/mm_utils.py +79 -7
- sglang/srt/model_executor/cuda_graph_runner.py +32 -8
- sglang/srt/model_executor/forward_batch_info.py +51 -26
- sglang/srt/model_executor/model_runner.py +201 -58
- sglang/srt/models/gemma2.py +10 -6
- sglang/srt/models/gpt_bigcode.py +1 -1
- sglang/srt/models/grok.py +11 -1
- sglang/srt/models/llama_embedding.py +4 -0
- sglang/srt/models/llava.py +176 -59
- sglang/srt/models/qwen2.py +9 -3
- sglang/srt/openai_api/adapter.py +200 -39
- sglang/srt/openai_api/protocol.py +2 -0
- sglang/srt/sampling/sampling_batch_info.py +136 -0
- sglang/srt/{sampling_params.py → sampling/sampling_params.py} +22 -0
- sglang/srt/server.py +92 -57
- sglang/srt/server_args.py +43 -15
- sglang/srt/utils.py +26 -16
- sglang/test/runners.py +22 -30
- sglang/test/simple_eval_common.py +9 -10
- sglang/test/simple_eval_gpqa.py +2 -1
- sglang/test/simple_eval_humaneval.py +2 -2
- sglang/test/simple_eval_math.py +2 -1
- sglang/test/simple_eval_mmlu.py +2 -1
- sglang/test/test_activation.py +55 -0
- sglang/test/test_utils.py +36 -53
- sglang/version.py +1 -1
- {sglang-0.2.13.dist-info → sglang-0.2.14.post1.dist-info}/METADATA +100 -27
- sglang-0.2.14.post1.dist-info/RECORD +114 -0
- {sglang-0.2.13.dist-info → sglang-0.2.14.post1.dist-info}/WHEEL +1 -1
- sglang/launch_server_llavavid.py +0 -29
- sglang-0.2.13.dist-info/RECORD +0 -112
- {sglang-0.2.13.dist-info → sglang-0.2.14.post1.dist-info}/LICENSE +0 -0
- {sglang-0.2.13.dist-info → sglang-0.2.14.post1.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,114 @@
|
|
1
|
+
sglang/__init__.py,sha256=T8MYdFfKFPZcgFKHMBpOCIlFbhjwmr77Nqm6mdE6bCY,1590
|
2
|
+
sglang/api.py,sha256=8B_ADgLN2fjo9Ej123hInfHA4wmpUkV0yyErSiRnfAA,6408
|
3
|
+
sglang/bench_latency.py,sha256=VEdGBX5vZSngS8AeOdJJRW65BIJsZXhKwAK5z20SZoI,16344
|
4
|
+
sglang/bench_serving.py,sha256=J_mMwnmDn0Jt07mzdGAuYOxpockHPLYJFL-kwoaqASY,36527
|
5
|
+
sglang/check_env.py,sha256=rGRABCgt-0SfUrow4px28b2P59aMn8eVTnN5eZc_a8s,5397
|
6
|
+
sglang/global_config.py,sha256=nwOjUflwqLQySPUMvk8Hk63TIS6mknh_ODSW3CZ1rJw,1704
|
7
|
+
sglang/launch_server.py,sha256=FODfO0DW546dh-u1qDlWtrhsmj6hxkarXXv3cIdgkj8,549
|
8
|
+
sglang/utils.py,sha256=zFYGkC4vOUR3sTv1TmQXcsOLZDtDBR3wnjqnDp3xMIs,8352
|
9
|
+
sglang/version.py,sha256=-MR2TfhYp1Y9iHDXSv_cRGYS5Mzv-azCfIoLfZjKpgg,29
|
10
|
+
sglang/lang/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
11
|
+
sglang/lang/chat_template.py,sha256=uqI_I9zIKXGXg7-W-yjqvx1ZeS_TuwFCms6wkmC2QmY,13411
|
12
|
+
sglang/lang/choices.py,sha256=-W1DVw9N9ZliVpvmWrzIXG4cswAah8eMQrHWzkS3D8o,6234
|
13
|
+
sglang/lang/compiler.py,sha256=o1C6G3TzhjSlsH-doTPy5oiVehr57dxNTa5oZw5TTAI,7639
|
14
|
+
sglang/lang/interpreter.py,sha256=-9VjAb5JqlxtBuQUDT08Cj2BW8VbLxTmJACe2cqza-s,30215
|
15
|
+
sglang/lang/ir.py,sha256=GRcPsEjnR4k5q5Kf-Rb2YgDBseCTGQoasclhjmQtL8Y,17511
|
16
|
+
sglang/lang/tracer.py,sha256=borJmlSJOhg1RUndGRnilnR60eEZz2Y9aU7BpftsOxU,8287
|
17
|
+
sglang/lang/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
18
|
+
sglang/lang/backend/anthropic.py,sha256=EXRX7xJgA5KZszX7toSLVnKzFQ5EO0Loj-YjHFtxSxg,2081
|
19
|
+
sglang/lang/backend/base_backend.py,sha256=Q5HdiDtyBewQeoYH0kDtBRVL8KFiEPNq9dw7XmauHQ8,1985
|
20
|
+
sglang/lang/backend/litellm.py,sha256=ugmL7sfUxkUHVbHtwNzHgdQAEd4UCjNQboFuE3KThcY,2450
|
21
|
+
sglang/lang/backend/openai.py,sha256=qM7eVH_kMxnDd2rpxOH0v76KxtOJFlAwgLgWIKvFGCI,15060
|
22
|
+
sglang/lang/backend/runtime_endpoint.py,sha256=SDlp03EuQEK1eGK4_IaFySWgxlp4wCs3EPewZ6O640E,9549
|
23
|
+
sglang/lang/backend/vertexai.py,sha256=O-iBLD-y3vq80UxnrAoJri7bxpgd-_eakZ88Cf8bEGA,4855
|
24
|
+
sglang/srt/conversation.py,sha256=Ze2_dTHG6jc04ti7vuOEnoEe1ehvhxCJRpa4EYD0T_8,18494
|
25
|
+
sglang/srt/hf_transformers_utils.py,sha256=OP5uBwnWiam6h9QvkBaG-nrDgkEUEwLXy1IWvW7rrRo,11737
|
26
|
+
sglang/srt/mm_utils.py,sha256=zox644S3IHUWmADdK4MnIbdTS2DWHOy0_Dq0gCU38QQ,12273
|
27
|
+
sglang/srt/model_config.py,sha256=k4OfRV-szWkFaJMIC40JoJGJ75AfYQ2hf4M1dS1aQ-o,6366
|
28
|
+
sglang/srt/server.py,sha256=UE2Vc7abITVe-yhe9WpnCCa_FdeDPUY8462uSpGDutA,19807
|
29
|
+
sglang/srt/server_args.py,sha256=GiDyPWCvYA_98mSE9LuvUoEodo9gRnNPPIPn0nFkxUs,18259
|
30
|
+
sglang/srt/utils.py,sha256=x9MdBu0e8HAgaNIGuxiMVL7_nh03kl_rWuMnLas_Dgo,24327
|
31
|
+
sglang/srt/constrained/__init__.py,sha256=NLpZGj9RIx83ejDrM_pfaRtqGgaPq_ggJszPQENUJ2E,2037
|
32
|
+
sglang/srt/constrained/base_tool_cache.py,sha256=5sazBMHHDpHMoqOjuY6itCxwTmIFCflIWEDXMtmrPVs,2006
|
33
|
+
sglang/srt/constrained/fsm_cache.py,sha256=1gvJzRxt6N_4KM1phlrFGPBquYiOUs_d8XIc7wztVk8,3100
|
34
|
+
sglang/srt/constrained/jump_forward.py,sha256=LWRsmGPQcH6KT87wXwCRqtblU3pcAVCEzO0nWPxevs0,6636
|
35
|
+
sglang/srt/layers/activation.py,sha256=CSKmMI2vLwLZL3heWdkbd3vYWzerIdmlvKJGKdu5kgk,4581
|
36
|
+
sglang/srt/layers/decode_attention.py,sha256=TPD_608ZX9fQ_HDImifkxG_qcEYmimbEYY8lCBIjFuM,16628
|
37
|
+
sglang/srt/layers/extend_attention.py,sha256=h4O0R7PJpAVKS3Vx_583zhrFPD0vv6XqzvOcHBI3zoc,14268
|
38
|
+
sglang/srt/layers/layernorm.py,sha256=RzN4eESN9S8mw32r2Nxarq7wKFdeG1yhxPmehUMx79s,2073
|
39
|
+
sglang/srt/layers/logits_processor.py,sha256=1hrxgvV1_qDsHiDak1JH6Vhn-hmkzi3AberLQ6-qkWw,13081
|
40
|
+
sglang/srt/layers/pooler.py,sha256=qNMG3Ycvt2yf9mk1Lcs-2K7oPeCuVeDYoHAxkMu9b_Q,1610
|
41
|
+
sglang/srt/layers/prefill_attention.py,sha256=y7vdcuX8lMa9Qf_jQYNDvQO9PVCBQSs3hb5LV2DFgpU,5256
|
42
|
+
sglang/srt/layers/radix_attention.py,sha256=o5a8r3XQ-oRwaxBlAgzJGv7p3dMbu0LrYsDc4uvpPgA,8338
|
43
|
+
sglang/srt/layers/sampler.py,sha256=XC6JcpuE1jO794u9AGRgzQ8xuu99G0l8PZKbX3_he7o,3995
|
44
|
+
sglang/srt/layers/fused_moe/__init__.py,sha256=bWCrDdOy2ANEXTb8CHYO63O3Iu3eZnn0PJbgl0z5vvE,75
|
45
|
+
sglang/srt/layers/fused_moe/fused_moe.py,sha256=1WM2cObWXcFWtqh_utGJFPnrT344rORwuQ9hJDaH2s0,23104
|
46
|
+
sglang/srt/layers/fused_moe/layer.py,sha256=GT3r2UPx_PAufJd0SUMOXyh76ymAeYDubd0SM0H71bo,20977
|
47
|
+
sglang/srt/managers/controller_multi.py,sha256=R45ST6oBlIwfUwuibMw0sgTk8iqphb_rFyIdW048JA4,6472
|
48
|
+
sglang/srt/managers/controller_single.py,sha256=tnc71OTe8KDYouMdfqgwBT4lX5nZt6Rak9t2GmKtAME,5119
|
49
|
+
sglang/srt/managers/detokenizer_manager.py,sha256=yQkL5gLomLiy1qc6e9HNz8hcj7JQFHm1AfIrzpXaWJE,6852
|
50
|
+
sglang/srt/managers/io_struct.py,sha256=4Cs655K4n_F_usu6R3YE5_RdcE0XO9AXQNk5vl2II2c,10534
|
51
|
+
sglang/srt/managers/policy_scheduler.py,sha256=7HNUxBKJE444s_bHcPpbnHCygsnH-NIXYNSC2q6mRmc,8584
|
52
|
+
sglang/srt/managers/schedule_batch.py,sha256=hkCf0IGxZanXLMSwfg5lub0NFNRGuykumAUtCPeryBU,25578
|
53
|
+
sglang/srt/managers/tokenizer_manager.py,sha256=aaZV7G3-m35pba1meRapqO7bdPjM2Cmkue5lbR_Jv3M,28836
|
54
|
+
sglang/srt/managers/tp_worker.py,sha256=-icuKdRrfb51hx8BsmQUlBQVpsvWlZ34Z2qIdPZSLrM,36433
|
55
|
+
sglang/srt/mem_cache/base_prefix_cache.py,sha256=qEQwEkG4E5rab2ZoTqcesf5pR_J4nV2jBxIHsBJHtIM,924
|
56
|
+
sglang/srt/mem_cache/chunk_cache.py,sha256=CjZZYlqQzq7mYOiBMLWA5XNb6HIyh5lIMdY-K0OUZEc,2368
|
57
|
+
sglang/srt/mem_cache/flush_cache.py,sha256=pTLKPRB17U6vl5RFJJvuJ4jCL2SyomgkUBNlkDpGRqo,978
|
58
|
+
sglang/srt/mem_cache/memory_pool.py,sha256=4br3Ea2bfA-YsF_sPOVHlF2zQzYGd8fVaYTp197yZsE,7871
|
59
|
+
sglang/srt/mem_cache/radix_cache.py,sha256=0AVr1BKKDOtTyybUkwxrz6PT8khDx-DpzgN5MgL27IE,10088
|
60
|
+
sglang/srt/model_executor/cuda_graph_runner.py,sha256=X4sC6xCnjsYjaM8YStyu2_IDIMbD1Muntd5HO1W-wiw,12011
|
61
|
+
sglang/srt/model_executor/forward_batch_info.py,sha256=Pv4vMXvCyJzRku2EHSVaNng28qglZQeska9wVkKUhyE,15544
|
62
|
+
sglang/srt/model_executor/model_runner.py,sha256=rXyGUQaTZDyi_AU3QLY5mRIf-h0yzJrtJ1PtnCuyxNM,24064
|
63
|
+
sglang/srt/models/chatglm.py,sha256=aoEgA2nflcOCIKtZojhUoboqxSP6i5IrrvuDOpzNPnE,13844
|
64
|
+
sglang/srt/models/commandr.py,sha256=2rAXRZRb4PkJZ4NWEqP_rIgsjxbdZyHpuoMOarqTWzQ,14163
|
65
|
+
sglang/srt/models/dbrx.py,sha256=N_0Ku_p1NCsc29NktUBNqPv7Z33XhYxOZK5xN7nzW4s,14661
|
66
|
+
sglang/srt/models/deepseek.py,sha256=7UJgde1EV9ey6d-CKRcEyTKh1_WhZdatpZiltIuqpik,16006
|
67
|
+
sglang/srt/models/deepseek_v2.py,sha256=uk--2a1e83H6U9wTx_wd3UvkS3VrSRSkjCOjky0R0uo,27004
|
68
|
+
sglang/srt/models/gemma.py,sha256=3orOUznoGt2NxVKO5c8AjD_ue0gWqwb7LnKbhlcS5Vg,12276
|
69
|
+
sglang/srt/models/gemma2.py,sha256=z-zhvLy3CX6OA_LF-lJSjpbN6dlyqFrHG4Gai7l0eM0,16584
|
70
|
+
sglang/srt/models/gpt_bigcode.py,sha256=kzHYogeGXZF4KHpkXA-RGqvs016mA-6klWxD2QJTi9E,10195
|
71
|
+
sglang/srt/models/grok.py,sha256=mdzd-fz8FirICdcpYLD94vwY3_ca0lxpIze8fAqDbHg,14805
|
72
|
+
sglang/srt/models/internlm2.py,sha256=6j7JH0p3yib8GZDH8Cmrs-pgwfH3eOlAK6V3Cq64O7w,12202
|
73
|
+
sglang/srt/models/llama2.py,sha256=HmzE1I8OnesmrdPY5b56l7okhWH_lRvWAg16K-UwKHg,14300
|
74
|
+
sglang/srt/models/llama_classification.py,sha256=Dvzy3PfETiJtnKFOk8qDDLUoZECf_cpSrNeA60PaDo4,4932
|
75
|
+
sglang/srt/models/llama_embedding.py,sha256=NQCQ3MnK3iRohL-UdY5UWxW4LlZ3RQZ7w4mlFOnpVrM,3696
|
76
|
+
sglang/srt/models/llava.py,sha256=iuXLJVDWBiYo8zJuDPSSjt2LYqbkg2MAcOFUZO1fOX4,24353
|
77
|
+
sglang/srt/models/llavavid.py,sha256=MX7YpqYh5J4BoOnV7vVAIfoOlBFQXYpp8Kpe7WK0ejk,13562
|
78
|
+
sglang/srt/models/minicpm.py,sha256=ioqCsTCE_oF8xqGF5fm5cK9dclK5Y0EQ1UJfyteIDDo,13825
|
79
|
+
sglang/srt/models/mistral.py,sha256=jlrWBVNXbAUziAaIdHAjFcOJnKtn9Bl8rBd65ypJM-I,819
|
80
|
+
sglang/srt/models/mixtral.py,sha256=cZK-1kGXQC8ZC0tFNmbAoqWlyrrvv5omumpDdEwzzss,13623
|
81
|
+
sglang/srt/models/mixtral_quant.py,sha256=wMACJq78OTWj7HlqPDRNEh8cjrVAjKqJEsOG3CO5xow,14072
|
82
|
+
sglang/srt/models/qwen.py,sha256=ssdSgVuhT1Ei0JPa0xwqzrwwPNwkCHRJA4q70hK-Z7E,9988
|
83
|
+
sglang/srt/models/qwen2.py,sha256=RHzzHfrq3_VoYkjoJu10z0PtyQ_Mit3mHHAO07214q0,12577
|
84
|
+
sglang/srt/models/qwen2_moe.py,sha256=-Ijn_H2IGCjQAYA-9teS9IXKTPMBWSkkPp0Nox6MCuQ,17729
|
85
|
+
sglang/srt/models/stablelm.py,sha256=30ngpc0Xq3VxzXJlf6svP1oax8Q3krMJkxM8PVKtZWU,11359
|
86
|
+
sglang/srt/models/yivl.py,sha256=p4s_D_m4H2exP4b91Y-CTkq8T-eIG3DJsFy9pB0e7TM,4932
|
87
|
+
sglang/srt/openai_api/adapter.py,sha256=5k2YGX6rEwpYlF3LaDs5Qvh6vD25LhAnPKWhcj2vnT0,49385
|
88
|
+
sglang/srt/openai_api/protocol.py,sha256=73i1iFVI0tAJCLMV9QT4rgMzbyYoZ8-iuTIYs_Yx0Xo,9487
|
89
|
+
sglang/srt/sampling/sampling_batch_info.py,sha256=TJLKc-FXqrI3-hHKEf-Bfu-W8NxNOEkmNvPaTN1O-g4,4911
|
90
|
+
sglang/srt/sampling/sampling_params.py,sha256=ggOXxafqfCD-xrGYcM57byLZ79CIeBP4AD5F44L_CW0,5635
|
91
|
+
sglang/srt/sampling/penaltylib/__init__.py,sha256=5vQw0Y5DSzmsoFg1IdMIKLwFVhYZ5ArADHVBYbSmOec,513
|
92
|
+
sglang/srt/sampling/penaltylib/orchestrator.py,sha256=WkTNeDhj9H9rtp2ZZeX6MS2sdKSGlLboE6FcuKrwUo0,10815
|
93
|
+
sglang/srt/sampling/penaltylib/penalizers/frequency_penalty.py,sha256=IvYioX53Vq_ji-0Zhcz_r5mUa3T3GaIydVS6K4FhWfE,2557
|
94
|
+
sglang/srt/sampling/penaltylib/penalizers/min_new_tokens.py,sha256=XJZP0C4NFyXgcODbIWXxrgVEjmRgqLdZuVAtoN-LveY,3565
|
95
|
+
sglang/srt/sampling/penaltylib/penalizers/presence_penalty.py,sha256=0PlANTrR959foTA3Nj5qBE7ndaOZgG-9X6LhzlmEUc8,2533
|
96
|
+
sglang/srt/sampling/penaltylib/penalizers/repetition_penalty.py,sha256=v9jOgA0-I31WcrhIydiFbpy2ZJPLytFLGM98NRPd2sU,2820
|
97
|
+
sglang/test/run_eval.py,sha256=NWxeLWmInBgkCvC9Jr_QzF7GfAiBve3Gf1JQrEOlNlU,3899
|
98
|
+
sglang/test/runners.py,sha256=7bh1ApkK6LCouQJzFXi4s2woFvmIUkh6wDeQA2luGCY,7686
|
99
|
+
sglang/test/simple_eval_common.py,sha256=r0G-9QLycs2ax3RMc44T_61fzMxlpTzv6pececC7lyY,12379
|
100
|
+
sglang/test/simple_eval_gpqa.py,sha256=8Xt9Bw05c7SZTYrCZgB68OZUqUbLo69ywiyx0bTvSUk,3220
|
101
|
+
sglang/test/simple_eval_humaneval.py,sha256=7lTi841NT58smNOtRwCedrdX9IWWypdLkOtaQOBy-GI,5687
|
102
|
+
sglang/test/simple_eval_math.py,sha256=6kGKNwNbLN-Af3Wj8WTimWhH-Xp3enDmSvvSjsgWUpk,2550
|
103
|
+
sglang/test/simple_eval_mgsm.py,sha256=wfbqJW9Rkc66vzq2fEMF6jchmoA8mw1OUiGU55cZ2B0,10261
|
104
|
+
sglang/test/simple_eval_mmlu.py,sha256=FkwamjGMjueTixymkedF-YiPloSLiy4ftILFUrKZ9XI,4357
|
105
|
+
sglang/test/test_activation.py,sha256=jkdNRzJnbd5OgZliQaIXpxovlcky17UrweomcOcMxoE,1442
|
106
|
+
sglang/test/test_layernorm.py,sha256=VDdoeqGvebUa-l3rDiid6cC7wZq0Phpbm5fxxD0-cpg,1910
|
107
|
+
sglang/test/test_programs.py,sha256=V_-Bx3lLkw37P6gDyA7mZCqxlyNMaFLBkRrPMQQQqn4,14909
|
108
|
+
sglang/test/test_utils.py,sha256=HD-9rcj7EFS_NX1GQFU5613ITQlZaTK2l9RmqA0F7x4,14380
|
109
|
+
sglang/test/srt/sampling/penaltylib/utils.py,sha256=-0p0rV-P4lNo7xAe3rQSBHTubc50a-DFyOQmLGAkgkQ,12515
|
110
|
+
sglang-0.2.14.post1.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
111
|
+
sglang-0.2.14.post1.dist-info/METADATA,sha256=FstugZ3Zw82h3E_xf6fNBN9HjdMPF4kyXiQA_KpQ3ak,37163
|
112
|
+
sglang-0.2.14.post1.dist-info/WHEEL,sha256=UvcQYKBHoFqaQd6LKyqHw9fxEolWLQnlzP0h_LgJAfI,91
|
113
|
+
sglang-0.2.14.post1.dist-info/top_level.txt,sha256=yxhh3pYQkcnA7v3Bg889C2jZhvtJdEincysO7PEB09M,7
|
114
|
+
sglang-0.2.14.post1.dist-info/RECORD,,
|
sglang/launch_server_llavavid.py
DELETED
@@ -1,29 +0,0 @@
|
|
1
|
-
"""Launch the inference server for Llava-video model."""
|
2
|
-
|
3
|
-
import argparse
|
4
|
-
|
5
|
-
from sglang.srt.server import ServerArgs, launch_server
|
6
|
-
|
7
|
-
if __name__ == "__main__":
|
8
|
-
model_overide_args = {}
|
9
|
-
|
10
|
-
model_overide_args["mm_spatial_pool_stride"] = 2
|
11
|
-
model_overide_args["architectures"] = ["LlavaVidForCausalLM"]
|
12
|
-
model_overide_args["num_frames"] = 16
|
13
|
-
model_overide_args["model_type"] = "llavavid"
|
14
|
-
if model_overide_args["num_frames"] == 32:
|
15
|
-
model_overide_args["rope_scaling"] = {"factor": 2.0, "type": "linear"}
|
16
|
-
model_overide_args["max_sequence_length"] = 4096 * 2
|
17
|
-
model_overide_args["tokenizer_model_max_length"] = 4096 * 2
|
18
|
-
model_overide_args["model_max_length"] = 4096 * 2
|
19
|
-
|
20
|
-
parser = argparse.ArgumentParser()
|
21
|
-
ServerArgs.add_cli_args(parser)
|
22
|
-
args = parser.parse_args()
|
23
|
-
|
24
|
-
if "34b" in args.model_path.lower():
|
25
|
-
model_overide_args["image_token_index"] = 64002
|
26
|
-
|
27
|
-
server_args = ServerArgs.from_cli_args(args)
|
28
|
-
|
29
|
-
launch_server(server_args, model_overide_args, None)
|
sglang-0.2.13.dist-info/RECORD
DELETED
@@ -1,112 +0,0 @@
|
|
1
|
-
sglang/__init__.py,sha256=T8MYdFfKFPZcgFKHMBpOCIlFbhjwmr77Nqm6mdE6bCY,1590
|
2
|
-
sglang/api.py,sha256=sRuA17JzayE9SFOhaZFqKFJDb_aRpNlcyKiMA5BzsDk,6258
|
3
|
-
sglang/bench_latency.py,sha256=UM5noYvFb6hc7wS82WAFeWTx3u83vkg9pfhyW0KdvY4,16234
|
4
|
-
sglang/bench_serving.py,sha256=sS-fawAyzngrOVbPE3N1FBxPojoPd9vj9XQDsWpIYTQ,35798
|
5
|
-
sglang/check_env.py,sha256=oU8VmjjPK2SviRhr41cF1953soBu-eTT5E0Hf04zMzo,4974
|
6
|
-
sglang/global_config.py,sha256=nwOjUflwqLQySPUMvk8Hk63TIS6mknh_ODSW3CZ1rJw,1704
|
7
|
-
sglang/launch_server.py,sha256=Gg8CwNlTCCfg1dF65ZT9ePLxOT9LKtY79GhIPG6PCrU,358
|
8
|
-
sglang/launch_server_llavavid.py,sha256=40uaazMsavKuk6YXFa5v37kdUpFGuealgJJeph1g8gU,1025
|
9
|
-
sglang/utils.py,sha256=zFYGkC4vOUR3sTv1TmQXcsOLZDtDBR3wnjqnDp3xMIs,8352
|
10
|
-
sglang/version.py,sha256=C0atO05M0rfDTTHt02NxNa4jt0eSqXM4AxShEhb2epA,23
|
11
|
-
sglang/lang/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
12
|
-
sglang/lang/chat_template.py,sha256=psIlhaDo70twgLrx5Lgln03metLEA3-FZuixeI0Y7Ao,13309
|
13
|
-
sglang/lang/choices.py,sha256=-W1DVw9N9ZliVpvmWrzIXG4cswAah8eMQrHWzkS3D8o,6234
|
14
|
-
sglang/lang/compiler.py,sha256=1Tc6MQs4RsIfrNmmO7PMSUEHIqvNqKOp_HxaYqonwFE,7533
|
15
|
-
sglang/lang/interpreter.py,sha256=8QiLvjUgVJrtzIjS9lCUR01k7BeZWZQsmRAwLMz-cmA,30194
|
16
|
-
sglang/lang/ir.py,sha256=WOZdRbONMhhSeD75bvUeQRv4gObxVMtkvzmalRrVdkM,17261
|
17
|
-
sglang/lang/tracer.py,sha256=borJmlSJOhg1RUndGRnilnR60eEZz2Y9aU7BpftsOxU,8287
|
18
|
-
sglang/lang/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
19
|
-
sglang/lang/backend/anthropic.py,sha256=EXRX7xJgA5KZszX7toSLVnKzFQ5EO0Loj-YjHFtxSxg,2081
|
20
|
-
sglang/lang/backend/base_backend.py,sha256=Q5HdiDtyBewQeoYH0kDtBRVL8KFiEPNq9dw7XmauHQ8,1985
|
21
|
-
sglang/lang/backend/litellm.py,sha256=ugmL7sfUxkUHVbHtwNzHgdQAEd4UCjNQboFuE3KThcY,2450
|
22
|
-
sglang/lang/backend/openai.py,sha256=qM7eVH_kMxnDd2rpxOH0v76KxtOJFlAwgLgWIKvFGCI,15060
|
23
|
-
sglang/lang/backend/runtime_endpoint.py,sha256=SDlp03EuQEK1eGK4_IaFySWgxlp4wCs3EPewZ6O640E,9549
|
24
|
-
sglang/lang/backend/vertexai.py,sha256=O-iBLD-y3vq80UxnrAoJri7bxpgd-_eakZ88Cf8bEGA,4855
|
25
|
-
sglang/srt/conversation.py,sha256=V5YuoeO6-aLqGv0p3J2qx8TnBJbN1oTopYFutNul3GQ,16491
|
26
|
-
sglang/srt/hf_transformers_utils.py,sha256=Tf_RplcW7llVXsigRvSGqmeAUxBeAL8rPCkzuqWfZ8U,11925
|
27
|
-
sglang/srt/mm_utils.py,sha256=n7_GmbOM_0IWVXovpM34rKIBw0Py9yb_NXSQw27u4OA,9454
|
28
|
-
sglang/srt/model_config.py,sha256=k4OfRV-szWkFaJMIC40JoJGJ75AfYQ2hf4M1dS1aQ-o,6366
|
29
|
-
sglang/srt/sampling_params.py,sha256=CIrM-OLAjUJ8oSQfhXetjv50BAseexWYOV5Wr6LXYeY,4739
|
30
|
-
sglang/srt/server.py,sha256=gSGC6MJLLXsuusizKzTxJaaWiaQjsa-Zm5hxV2fYHb8,18845
|
31
|
-
sglang/srt/server_args.py,sha256=YoTVFzt65w1vjypyh0a4FV7BNreVGS49d8uf6TPrM_w,17083
|
32
|
-
sglang/srt/utils.py,sha256=MIDD53BT4ukaHO-zmEQZD5l7Xco_gefO0co4FJsMsn4,24053
|
33
|
-
sglang/srt/constrained/__init__.py,sha256=NLpZGj9RIx83ejDrM_pfaRtqGgaPq_ggJszPQENUJ2E,2037
|
34
|
-
sglang/srt/constrained/base_tool_cache.py,sha256=5sazBMHHDpHMoqOjuY6itCxwTmIFCflIWEDXMtmrPVs,2006
|
35
|
-
sglang/srt/constrained/fsm_cache.py,sha256=QTrBFoZCp2FeigtIakz2MCgQLtvQFXgl2lDPQaGtu9M,2784
|
36
|
-
sglang/srt/constrained/jump_forward.py,sha256=9_HxmXtWjr5S6a5e0cBimbY3ZhiLiJC74V6jIqDXfuo,6575
|
37
|
-
sglang/srt/layers/activation.py,sha256=j2zQmY1snfB5DqrYr5KqRUEkMXQn6LVnkeur60FfMCU,1175
|
38
|
-
sglang/srt/layers/decode_attention.py,sha256=Vgxd2rWzSZkNFp0bjZRAUAusG4bz6iy3D0CULnN-cdk,8904
|
39
|
-
sglang/srt/layers/extend_attention.py,sha256=h4O0R7PJpAVKS3Vx_583zhrFPD0vv6XqzvOcHBI3zoc,14268
|
40
|
-
sglang/srt/layers/layernorm.py,sha256=RzN4eESN9S8mw32r2Nxarq7wKFdeG1yhxPmehUMx79s,2073
|
41
|
-
sglang/srt/layers/logits_processor.py,sha256=wBgo6IVxWgV4vYRQesnuE2qA8ynB2oFtv0COZSAMIeA,11374
|
42
|
-
sglang/srt/layers/pooler.py,sha256=qNMG3Ycvt2yf9mk1Lcs-2K7oPeCuVeDYoHAxkMu9b_Q,1610
|
43
|
-
sglang/srt/layers/prefill_attention.py,sha256=y7vdcuX8lMa9Qf_jQYNDvQO9PVCBQSs3hb5LV2DFgpU,5256
|
44
|
-
sglang/srt/layers/radix_attention.py,sha256=EA7rc73ZGnle2tQlslF9Ri_VEY07jD0e0cPiKcsqOyA,8473
|
45
|
-
sglang/srt/layers/fused_moe/__init__.py,sha256=bWCrDdOy2ANEXTb8CHYO63O3Iu3eZnn0PJbgl0z5vvE,75
|
46
|
-
sglang/srt/layers/fused_moe/fused_moe.py,sha256=1WM2cObWXcFWtqh_utGJFPnrT344rORwuQ9hJDaH2s0,23104
|
47
|
-
sglang/srt/layers/fused_moe/layer.py,sha256=ByNlMmmXsckcsjI12rhlg_IH0KvO6zWJoOYuk7i4ogY,20947
|
48
|
-
sglang/srt/managers/controller_multi.py,sha256=LYI-XE9h57DW8Uh4gpd8upsC3p2dd5weKzddEH274jg,6626
|
49
|
-
sglang/srt/managers/controller_single.py,sha256=CdQ9_XPZdcWF5jArDmVR8K-WZ9_8Gpgk4SwANKxTX-Y,5112
|
50
|
-
sglang/srt/managers/detokenizer_manager.py,sha256=OXufjdCt2ebt-S7MDndjY9Ew16rP4fhualGgj6YEKp0,6295
|
51
|
-
sglang/srt/managers/io_struct.py,sha256=Xvfl6DNZ2Ek2S4qlRzpVo3foc-aC-1-N-5odcJ4gdq4,9446
|
52
|
-
sglang/srt/managers/policy_scheduler.py,sha256=KRFaZwjCAkPQDX3W8lbzrxYqgOe7LKFDj2BPlcmlnR8,8379
|
53
|
-
sglang/srt/managers/schedule_batch.py,sha256=L9kBQZBfsy-2Arzkx4ZjKjNL-zN1BErnv9LqRi3CQNI,30657
|
54
|
-
sglang/srt/managers/tokenizer_manager.py,sha256=4cf7JyuMGvLVp6Dv8pWG6c9285O6zuD2Ja0eEePUCNg,24857
|
55
|
-
sglang/srt/managers/tp_worker.py,sha256=TPtWHcLM-bh7GGdA7-8c-zdNLFeLxWNnl3iqODKwYWw,32583
|
56
|
-
sglang/srt/mem_cache/base_prefix_cache.py,sha256=qEQwEkG4E5rab2ZoTqcesf5pR_J4nV2jBxIHsBJHtIM,924
|
57
|
-
sglang/srt/mem_cache/chunk_cache.py,sha256=CjZZYlqQzq7mYOiBMLWA5XNb6HIyh5lIMdY-K0OUZEc,2368
|
58
|
-
sglang/srt/mem_cache/flush_cache.py,sha256=pTLKPRB17U6vl5RFJJvuJ4jCL2SyomgkUBNlkDpGRqo,978
|
59
|
-
sglang/srt/mem_cache/memory_pool.py,sha256=eXDCstd5Mvu1CbHt1y9z27Eq60QYwW45FsKbZspu4yw,5310
|
60
|
-
sglang/srt/mem_cache/radix_cache.py,sha256=0AVr1BKKDOtTyybUkwxrz6PT8khDx-DpzgN5MgL27IE,10088
|
61
|
-
sglang/srt/model_executor/cuda_graph_runner.py,sha256=xvhFptAJKonqnEjeVYaIiKwhEM4NzbSeF9YvC6YqVc8,11364
|
62
|
-
sglang/srt/model_executor/forward_batch_info.py,sha256=tcWwiKBU2W2USg19ASRlx-9utvYL6PTO0NPNyK5frJk,14272
|
63
|
-
sglang/srt/model_executor/model_runner.py,sha256=QpNzsV1WiH4_1T0klmM6GjivWI-fKLATC5E67C1LSYk,18158
|
64
|
-
sglang/srt/models/chatglm.py,sha256=aoEgA2nflcOCIKtZojhUoboqxSP6i5IrrvuDOpzNPnE,13844
|
65
|
-
sglang/srt/models/commandr.py,sha256=2rAXRZRb4PkJZ4NWEqP_rIgsjxbdZyHpuoMOarqTWzQ,14163
|
66
|
-
sglang/srt/models/dbrx.py,sha256=N_0Ku_p1NCsc29NktUBNqPv7Z33XhYxOZK5xN7nzW4s,14661
|
67
|
-
sglang/srt/models/deepseek.py,sha256=7UJgde1EV9ey6d-CKRcEyTKh1_WhZdatpZiltIuqpik,16006
|
68
|
-
sglang/srt/models/deepseek_v2.py,sha256=uk--2a1e83H6U9wTx_wd3UvkS3VrSRSkjCOjky0R0uo,27004
|
69
|
-
sglang/srt/models/gemma.py,sha256=3orOUznoGt2NxVKO5c8AjD_ue0gWqwb7LnKbhlcS5Vg,12276
|
70
|
-
sglang/srt/models/gemma2.py,sha256=IUXKjwO11dpnhevmapS9jz_qPZvzSKrHhYHIXnBR9AU,16475
|
71
|
-
sglang/srt/models/gpt_bigcode.py,sha256=OKk9UP67as3T5bePlTRGHTCD-1wqaUEk92AowXPm6dg,10204
|
72
|
-
sglang/srt/models/grok.py,sha256=TrYcCQZhV7f5SUntU4Lo4ZDC8uBi0Vg0SWtyYiZxdqs,14530
|
73
|
-
sglang/srt/models/internlm2.py,sha256=6j7JH0p3yib8GZDH8Cmrs-pgwfH3eOlAK6V3Cq64O7w,12202
|
74
|
-
sglang/srt/models/llama2.py,sha256=HmzE1I8OnesmrdPY5b56l7okhWH_lRvWAg16K-UwKHg,14300
|
75
|
-
sglang/srt/models/llama_classification.py,sha256=Dvzy3PfETiJtnKFOk8qDDLUoZECf_cpSrNeA60PaDo4,4932
|
76
|
-
sglang/srt/models/llama_embedding.py,sha256=e2lpZ6GHKrHT1rr7_5gHGoCpfqdOBMusZCz34n62lec,3542
|
77
|
-
sglang/srt/models/llava.py,sha256=-ysi192vpBDxNaMS8qaLOhC34lXQyRtbG_0niVaceSo,18436
|
78
|
-
sglang/srt/models/llavavid.py,sha256=MX7YpqYh5J4BoOnV7vVAIfoOlBFQXYpp8Kpe7WK0ejk,13562
|
79
|
-
sglang/srt/models/minicpm.py,sha256=ioqCsTCE_oF8xqGF5fm5cK9dclK5Y0EQ1UJfyteIDDo,13825
|
80
|
-
sglang/srt/models/mistral.py,sha256=jlrWBVNXbAUziAaIdHAjFcOJnKtn9Bl8rBd65ypJM-I,819
|
81
|
-
sglang/srt/models/mixtral.py,sha256=cZK-1kGXQC8ZC0tFNmbAoqWlyrrvv5omumpDdEwzzss,13623
|
82
|
-
sglang/srt/models/mixtral_quant.py,sha256=wMACJq78OTWj7HlqPDRNEh8cjrVAjKqJEsOG3CO5xow,14072
|
83
|
-
sglang/srt/models/qwen.py,sha256=ssdSgVuhT1Ei0JPa0xwqzrwwPNwkCHRJA4q70hK-Z7E,9988
|
84
|
-
sglang/srt/models/qwen2.py,sha256=eeah76x-OYZiy6Bb1SDNVk8m_xXHYuh-P58GXjEFZ4w,12266
|
85
|
-
sglang/srt/models/qwen2_moe.py,sha256=-Ijn_H2IGCjQAYA-9teS9IXKTPMBWSkkPp0Nox6MCuQ,17729
|
86
|
-
sglang/srt/models/stablelm.py,sha256=30ngpc0Xq3VxzXJlf6svP1oax8Q3krMJkxM8PVKtZWU,11359
|
87
|
-
sglang/srt/models/yivl.py,sha256=p4s_D_m4H2exP4b91Y-CTkq8T-eIG3DJsFy9pB0e7TM,4932
|
88
|
-
sglang/srt/openai_api/adapter.py,sha256=C53adcpLGfIUm_B259iWnOCQ3B3VjJbqFseqP8Vo-t8,43064
|
89
|
-
sglang/srt/openai_api/protocol.py,sha256=knf-nds0XO2LYg-hPM-Ho1f1y2XZIV_Gvg3xcCKLfgQ,9411
|
90
|
-
sglang/srt/sampling/penaltylib/__init__.py,sha256=5vQw0Y5DSzmsoFg1IdMIKLwFVhYZ5ArADHVBYbSmOec,513
|
91
|
-
sglang/srt/sampling/penaltylib/orchestrator.py,sha256=WkTNeDhj9H9rtp2ZZeX6MS2sdKSGlLboE6FcuKrwUo0,10815
|
92
|
-
sglang/srt/sampling/penaltylib/penalizers/frequency_penalty.py,sha256=IvYioX53Vq_ji-0Zhcz_r5mUa3T3GaIydVS6K4FhWfE,2557
|
93
|
-
sglang/srt/sampling/penaltylib/penalizers/min_new_tokens.py,sha256=XJZP0C4NFyXgcODbIWXxrgVEjmRgqLdZuVAtoN-LveY,3565
|
94
|
-
sglang/srt/sampling/penaltylib/penalizers/presence_penalty.py,sha256=0PlANTrR959foTA3Nj5qBE7ndaOZgG-9X6LhzlmEUc8,2533
|
95
|
-
sglang/srt/sampling/penaltylib/penalizers/repetition_penalty.py,sha256=v9jOgA0-I31WcrhIydiFbpy2ZJPLytFLGM98NRPd2sU,2820
|
96
|
-
sglang/test/run_eval.py,sha256=NWxeLWmInBgkCvC9Jr_QzF7GfAiBve3Gf1JQrEOlNlU,3899
|
97
|
-
sglang/test/runners.py,sha256=J4XfBSPhZvLiHLrDsHUuIKjX3kzbMrD7fFEPr07SUkU,7975
|
98
|
-
sglang/test/simple_eval_common.py,sha256=HL1bfgkTAKP7sk-kShg73WTeADhuBD6xSsuLbV_9C3s,12359
|
99
|
-
sglang/test/simple_eval_gpqa.py,sha256=CaRAuHdZj0m4mRm4tH9k7cB0kQxe0LHwlz7Vn1qyKps,3189
|
100
|
-
sglang/test/simple_eval_humaneval.py,sha256=iCtN2LBL6j3nxMDjRJ--m0MCNPAwDo81gJ2whE-2Rt0,5674
|
101
|
-
sglang/test/simple_eval_math.py,sha256=EQblQmtUt-kl558drzhP7c6KhpDNgr1EJhhKx5eeHM4,2519
|
102
|
-
sglang/test/simple_eval_mgsm.py,sha256=wfbqJW9Rkc66vzq2fEMF6jchmoA8mw1OUiGU55cZ2B0,10261
|
103
|
-
sglang/test/simple_eval_mmlu.py,sha256=KqSSdSu2qfoKQ870ttxev1NJ7c90xv2mvKOQsSODtAw,4326
|
104
|
-
sglang/test/test_layernorm.py,sha256=VDdoeqGvebUa-l3rDiid6cC7wZq0Phpbm5fxxD0-cpg,1910
|
105
|
-
sglang/test/test_programs.py,sha256=V_-Bx3lLkw37P6gDyA7mZCqxlyNMaFLBkRrPMQQQqn4,14909
|
106
|
-
sglang/test/test_utils.py,sha256=Fw606sa8sTX6HJ7OCuyDUH8LQr9PvtwBKYnyZj2SLWU,14741
|
107
|
-
sglang/test/srt/sampling/penaltylib/utils.py,sha256=-0p0rV-P4lNo7xAe3rQSBHTubc50a-DFyOQmLGAkgkQ,12515
|
108
|
-
sglang-0.2.13.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
109
|
-
sglang-0.2.13.dist-info/METADATA,sha256=oy69SBbn-iEZE0JRzPkHuhzRlAjNj6v8twSXrjsOWXs,34892
|
110
|
-
sglang-0.2.13.dist-info/WHEEL,sha256=HiCZjzuy6Dw0hdX5R3LCFPDmFS4BWl8H-8W39XfmgX4,91
|
111
|
-
sglang-0.2.13.dist-info/top_level.txt,sha256=yxhh3pYQkcnA7v3Bg889C2jZhvtJdEincysO7PEB09M,7
|
112
|
-
sglang-0.2.13.dist-info/RECORD,,
|
File without changes
|
File without changes
|