vec-inf 0.6.0__py3-none-any.whl → 0.6.1__py3-none-any.whl

vec_inf/cli/_cli.py

@@ -18,6 +18,7 @@ metrics
     Stream real-time performance metrics
 """
 
+import json
 import time
 from typing import Optional, Union
 
@@ -72,6 +73,21 @@ def cli() -> None:
     type=str,
     help="Quality of service",
 )
+@click.option(
+    "--exclude",
+    type=str,
+    help="Exclude certain nodes from the resources granted to the job",
+)
+@click.option(
+    "--node-list",
+    type=str,
+    help="Request a specific list of nodes for deployment",
+)
+@click.option(
+    "--bind",
+    type=str,
+    help="Additional binds for the singularity container as a comma separated list of bind paths",
+)
 @click.option(
     "--time",
     type=str,
@@ -124,8 +140,16 @@ def launch(
             Number of nodes to use
         - gpus_per_node : int, optional
             Number of GPUs per node
+        - account : str, optional
+            Charge resources used by this job to specified account
         - qos : str, optional
             Quality of service tier
+        - exclude : str, optional
+            Exclude certain nodes from the resources granted to the job
+        - node_list : str, optional
+            Request a specific list of nodes for deployment
+        - bind : str, optional
+            Additional binds for the singularity container
         - time : str, optional
             Time limit for job
         - venv : str, optional
@@ -157,8 +181,9 @@ def launch(
 
         # Display launch information
         launch_formatter = LaunchResponseFormatter(model_name, launch_response.config)
+
         if json_mode:
-            click.echo(launch_response.config)
+            click.echo(json.dumps(launch_response.config))
         else:
             launch_info_table = launch_formatter.format_table_output()
             CONSOLE.print(launch_info_table)

vec_inf/client/_client_vars.py

@@ -21,7 +21,12 @@ SLURM_JOB_CONFIG_ARGS : dict
 from pathlib import Path
 from typing import TypedDict
 
-from vec_inf.client.slurm_vars import SINGULARITY_LOAD_CMD
+from vec_inf.client.slurm_vars import (
+    LD_LIBRARY_PATH,
+    SINGULARITY_IMAGE,
+    SINGULARITY_LOAD_CMD,
+    VLLM_NCCL_SO_PATH,
+)
 
 
 MODEL_READY_SIGNATURE = "INFO:     Application startup complete."
@@ -60,6 +65,8 @@ SLURM_JOB_CONFIG_ARGS = {
     "qos": "qos",
     "time": "time",
     "nodes": "num_nodes",
+    "exclude": "exclude",
+    "nodelist": "node_list",
     "gpus-per-node": "gpus_per_node",
     "cpus-per-task": "cpus_per_task",
     "mem": "mem_per_node",
@@ -71,7 +78,12 @@ SLURM_JOB_CONFIG_ARGS = {
 VLLM_SHORT_TO_LONG_MAP = {
     "-tp": "--tensor-parallel-size",
     "-pp": "--pipeline-parallel-size",
+    "-dp": "--data-parallel-size",
+    "-dpl": "--data-parallel-size-local",
+    "-dpa": "--data-parallel-address",
+    "-dpp": "--data-parallel-rpc-port",
     "-O": "--compilation-config",
+    "-q": "--quantization",
 }
 
 
@@ -117,6 +129,8 @@ class SlurmScriptTemplate(TypedDict):
         Commands for Singularity container setup
     imports : str
         Import statements and source commands
+    env_vars : list[str]
+        Environment variables to set
     singularity_command : str
         Template for Singularity execution command
     activate_venv : str
@@ -134,6 +148,7 @@ class SlurmScriptTemplate(TypedDict):
     shebang: ShebangConfig
     singularity_setup: list[str]
     imports: str
+    env_vars: list[str]
     singularity_command: str
     activate_venv: str
     server_setup: ServerSetupConfig
@@ -152,10 +167,14 @@ SLURM_SCRIPT_TEMPLATE: SlurmScriptTemplate = {
     },
     "singularity_setup": [
         SINGULARITY_LOAD_CMD,
-        "singularity exec {singularity_image} ray stop",
+        f"singularity exec {SINGULARITY_IMAGE} ray stop",
     ],
     "imports": "source {src_dir}/find_port.sh",
-    "singularity_command": "singularity exec --nv --bind {model_weights_path}:{model_weights_path} --containall {singularity_image}",
+    "env_vars": [
+        f"export LD_LIBRARY_PATH={LD_LIBRARY_PATH}",
+        f"export VLLM_NCCL_SO_PATH={VLLM_NCCL_SO_PATH}",
+    ],
+    "singularity_command": f"singularity exec --nv --bind {{model_weights_path}}{{additional_binds}} --containall {SINGULARITY_IMAGE}",
     "activate_venv": "source {venv}/bin/activate",
     "server_setup": {
         "single_node": [
@@ -203,8 +222,7 @@ SLURM_SCRIPT_TEMPLATE: SlurmScriptTemplate = {
         '    && mv temp.json "$json_path"',
     ],
     "launch_cmd": [
-        "python3.10 -m vllm.entrypoints.openai.api_server \\",
-        "    --model {model_weights_path} \\",
+        "vllm serve {model_weights_path} \\",
         "    --served-model-name {model_name} \\",
         '    --host "0.0.0.0" \\',
         "    --port $vllm_port_number \\",

vec_inf/client/_helper.py

@@ -5,7 +5,6 @@ metrics collection, and model registry operations.
 """
 
 import json
-import os
 import time
 import warnings
 from pathlib import Path
@@ -36,10 +35,6 @@ from vec_inf.client.models import (
     ModelType,
     StatusResponse,
 )
-from vec_inf.client.slurm_vars import (
-    LD_LIBRARY_PATH,
-    VLLM_NCCL_SO_PATH,
-)
 
 
 class ModelLauncher:
@@ -230,11 +225,6 @@ class ModelLauncher:
 
         return params
 
-    def _set_env_vars(self) -> None:
-        """Set environment variables for the launch command."""
-        os.environ["LD_LIBRARY_PATH"] = LD_LIBRARY_PATH
-        os.environ["VLLM_NCCL_SO_PATH"] = VLLM_NCCL_SO_PATH
-
     def _build_launch_command(self) -> str:
         """Generate the slurm script and construct the launch command.
 
@@ -259,9 +249,6 @@ class ModelLauncher:
         SlurmJobError
             If SLURM job submission fails
         """
-        # Set environment variables
-        self._set_env_vars()
-
         # Build and execute the launch command
         command_output, stderr = utils.run_bash_command(self._build_launch_command())
 

vec_inf/client/_slurm_script_generator.py

@@ -12,7 +12,6 @@ from vec_inf.client._client_vars import (
     SLURM_JOB_CONFIG_ARGS,
     SLURM_SCRIPT_TEMPLATE,
 )
-from vec_inf.client.slurm_vars import SINGULARITY_IMAGE
 
 
 class SlurmScriptGenerator:
@@ -40,6 +39,9 @@ class SlurmScriptGenerator:
         self.params = params
         self.is_multinode = int(self.params["num_nodes"]) > 1
         self.use_singularity = self.params["venv"] == "singularity"
+        self.additional_binds = self.params.get("bind", "")
+        if self.additional_binds:
+            self.additional_binds = f" --bind {self.additional_binds}"
         self.model_weights_path = str(
             Path(params["model_weights_parent_dir"], params["model_name"])
         )
@@ -87,11 +89,8 @@ class SlurmScriptGenerator:
         """
         server_script = ["\n"]
         if self.use_singularity:
-            server_script.append(
-                "\n".join(SLURM_SCRIPT_TEMPLATE["singularity_setup"]).format(
-                    singularity_image=SINGULARITY_IMAGE,
-                )
-            )
+            server_script.append("\n".join(SLURM_SCRIPT_TEMPLATE["singularity_setup"]))
+        server_script.append("\n".join(SLURM_SCRIPT_TEMPLATE["env_vars"]))
         server_script.append(
             SLURM_SCRIPT_TEMPLATE["imports"].format(src_dir=self.params["src_dir"])
         )
@@ -104,7 +103,7 @@ class SlurmScriptGenerator:
                     "SINGULARITY_PLACEHOLDER",
                     SLURM_SCRIPT_TEMPLATE["singularity_command"].format(
                         model_weights_path=self.model_weights_path,
-                        singularity_image=SINGULARITY_IMAGE,
+                        additional_binds=self.additional_binds,
                     ),
                 )
         else:
@@ -136,7 +135,7 @@ class SlurmScriptGenerator:
             launcher_script.append(
                 SLURM_SCRIPT_TEMPLATE["singularity_command"].format(
                     model_weights_path=self.model_weights_path,
-                    singularity_image=SINGULARITY_IMAGE,
+                    additional_binds=self.additional_binds,
                 )
                 + " \\"
             )

vec_inf/client/config.py

@@ -108,6 +108,16 @@ class ModelConfig(BaseModel):
     partition: Union[PARTITION, str] = Field(
         default=cast(str, DEFAULT_ARGS["partition"]), description="GPU partition type"
     )
+    exclude: Optional[str] = Field(
+        default=None,
+        description="Exclude certain nodes from the resources granted to the job",
+    )
+    node_list: Optional[str] = Field(
+        default=None, description="Request a specific list of nodes for deployment"
+    )
+    bind: Optional[str] = Field(
+        default=None, description="Additional binds for the singularity container"
+    )
     venv: str = Field(
         default="singularity", description="Virtual environment/container system"
     )

vec_inf/client/models.py

@@ -170,6 +170,12 @@ class LaunchOptions:
         Quality of Service level
     time : str, optional
         Time limit for the job
+    exclude : str, optional
+        Exclude certain nodes from the resources granted to the job
+    node_list : str, optional
+        Request a specific list of nodes for deployment
+    bind : str, optional
+        Additional binds for the singularity container
     vocab_size : int, optional
         Size of model vocabulary
     data_type : str, optional
@@ -191,6 +197,9 @@ class LaunchOptions:
     gpus_per_node: Optional[int] = None
     account: Optional[str] = None
     qos: Optional[str] = None
+    exclude: Optional[str] = None
+    node_list: Optional[str] = None
+    bind: Optional[str] = None
     time: Optional[str] = None
     vocab_size: Optional[int] = None
     data_type: Optional[str] = None

vec_inf/config/models.yaml

@@ -14,7 +14,6 @@ models:
       --tensor-parallel-size: 4
       --max-model-len: 8192
       --max-num-seqs: 256
-      --compilation-config: 3
   c4ai-command-r-plus-08-2024:
     model_family: c4ai-command-r
     model_variant: plus-08-2024
@@ -30,7 +29,6 @@ models:
       --tensor-parallel-size: 4
       --max-model-len: 65536
       --max-num-seqs: 256
-      --compilation-config: 3
   c4ai-command-r-08-2024:
     model_family: c4ai-command-r
     model_variant: 08-2024
@@ -494,7 +492,6 @@ models:
       --tensor-parallel-size: 4
       --max-model-len: 16384
       --max-num-seqs: 256
-      --compilation-config: 3
   Mistral-7B-Instruct-v0.1:
     model_family: Mistral
     model_variant: 7B-Instruct-v0.1
@@ -566,7 +563,6 @@ models:
       --tensor-parallel-size: 4
       --max-model-len: 32768
       --max-num-seqs: 256
-      --compilation-config: 3
   Mistral-Large-Instruct-2411:
     model_family: Mistral
     model_variant: Large-Instruct-2411
@@ -582,7 +578,6 @@ models:
       --tensor-parallel-size: 4
       --max-model-len: 32768
       --max-num-seqs: 256
-      --compilation-config: 3
   Mixtral-8x7B-Instruct-v0.1:
     model_family: Mixtral
     model_variant: 8x7B-Instruct-v0.1
@@ -613,7 +608,6 @@ models:
       --tensor-parallel-size: 4
       --max-model-len: 65536
       --max-num-seqs: 256
-      --compilation-config: 3
   Mixtral-8x22B-Instruct-v0.1:
     model_family: Mixtral
     model_variant: 8x22B-Instruct-v0.1
@@ -629,7 +623,6 @@ models:
       --tensor-parallel-size: 4
       --max-model-len: 65536
       --max-num-seqs: 256
-      --compilation-config: 3
   Phi-3-medium-128k-instruct:
     model_family: Phi-3
     model_variant: medium-128k-instruct

{vec_inf-0.6.0.dist-info → vec_inf-0.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vec-inf
-Version: 0.6.0
+Version: 0.6.1
 Summary: Efficient LLM inference on Slurm clusters using vLLM.
 Author-email: Marshall Wang <marshall.wang@vectorinstitute.ai>
 License-Expression: MIT
@@ -29,6 +29,7 @@ Description-Content-Type: text/markdown
 [![code checks](https://github.com/VectorInstitute/vector-inference/actions/workflows/code_checks.yml/badge.svg)](https://github.com/VectorInstitute/vector-inference/actions/workflows/code_checks.yml)
 [![docs](https://github.com/VectorInstitute/vector-inference/actions/workflows/docs.yml/badge.svg)](https://github.com/VectorInstitute/vector-inference/actions/workflows/docs.yml)
 [![codecov](https://codecov.io/github/VectorInstitute/vector-inference/branch/main/graph/badge.svg?token=NI88QSIGAC)](https://app.codecov.io/github/VectorInstitute/vector-inference/tree/main)
+[![vLLM](https://img.shields.io/badge/vllm-0.8.5.post1-blue)](https://docs.vllm.ai/en/v0.8.5.post1/index.html)
 ![GitHub License](https://img.shields.io/github/license/VectorInstitute/vector-inference)
 
 This repository provides an easy-to-use solution to run inference servers on [Slurm](https://slurm.schedmd.com/overview.html)-managed computing clusters using [vLLM](https://docs.vllm.ai/en/latest/). **All scripts in this repository runs natively on the Vector Institute cluster environment**. To adapt to other environments, update the environment variables in [`vec_inf/client/slurm_vars.py`](vec_inf/client/slurm_vars.py), and the model config for cached model weights in [`vec_inf/config/models.yaml`](vec_inf/config/models.yaml) accordingly.
@@ -39,7 +40,7 @@ If you are using the Vector cluster environment, and you don't need any customiz
 ```bash
 pip install vec-inf
 ```
-Otherwise, we recommend using the provided [`Dockerfile`](Dockerfile) to set up your own environment with the package
+Otherwise, we recommend using the provided [`Dockerfile`](Dockerfile) to set up your own environment with the package. The latest image has `vLLM` version `0.8.5.post1`.
 
 ## Usage
 
@@ -107,7 +108,7 @@ models:
     vllm_args:
       --max-model-len: 1010000
       --max-num-seqs: 256
-      --compilation-confi: 3
+      --compilation-config: 3
 ```
 
 You would then set the `VEC_INF_CONFIG` path using:
@@ -116,7 +117,11 @@ You would then set the `VEC_INF_CONFIG` path using:
 export VEC_INF_CONFIG=/h/<username>/my-model-config.yaml
 ```
 
-Note that there are other parameters that can also be added to the config but not shown in this example, check the [`ModelConfig`](vec_inf/client/config.py) for details.
+**NOTE**
+* There are other parameters that can also be added to the config but not shown in this example, check the [`ModelConfig`](vec_inf/client/config.py) for details.
+* Check [vLLM Engine Arguments](https://docs.vllm.ai/en/stable/serving/engine_args.html) for the full list of available vLLM engine arguments, the default parallel size for any parallelization is default to 1, so none of the sizes were set specifically in this example
+* For GPU partitions with non-Ampere architectures, e.g. `rtx6000`, `t4v2`, BF16 isn't supported. For models that have BF16 as the default type, when using a non-Ampere GPU, use FP16 instead, i.e. `--dtype: float16`
+* Setting `--compilation-config` to `3` currently breaks multi-node model launches, so we don't set them for models that require multiple nodes of GPUs.
 
 #### Other commands
 
@@ -182,8 +187,9 @@ Once the inference server is ready, you can start sending in inference requests.
     },
     "prompt_logprobs":null
 }
+
 ```
-**NOTE**: For multimodal models, currently only `ChatCompletion` is available, and only one image can be provided for each prompt.
+**NOTE**: Certain models don't adhere to OpenAI's chat template, e.g. Mistral family. For these models, you can either change your prompt to follow the model's default chat template or provide your own chat template via `--chat-template: TEMPLATE_PATH`.
 
 ## SSH tunnel from your local device
 If you want to run inference from your local device, you can open a SSH tunnel to your cluster environment like the following:

{vec_inf-0.6.0.dist-info → vec_inf-0.6.1.dist-info}/RECORD

@@ -2,24 +2,24 @@ vec_inf/README.md,sha256=3ocJHfV3kRftXFUCdHw3B-p4QQlXuNqkHnjPPNkCgfM,543
 vec_inf/__init__.py,sha256=bHwSIz9lebYuxIemni-lP0h3gwJHVbJnwExQKGJWw_Q,23
 vec_inf/find_port.sh,sha256=bGQ6LYSFVSsfDIGatrSg5YvddbZfaPL0R-Bjo4KYD6I,1088
 vec_inf/cli/__init__.py,sha256=5XIvGQCOnaGl73XMkwetjC-Ul3xuXGrWDXdYJ3aUzvU,27
-vec_inf/cli/_cli.py,sha256=bqyLvFK4Vqoh-wAaUPg50_qYbrW-c9Cl_-YySgVk5_M,9871
+vec_inf/cli/_cli.py,sha256=pqZeQr5WxAsV7KSYcUnx_mRL7RnHWk1zf9CcW_ct5uI,10663
 vec_inf/cli/_helper.py,sha256=i1QvJeIT3z7me6bv2Vot5c3NY555Dgo3q8iRlxhOlZ4,13047
 vec_inf/cli/_utils.py,sha256=23vSbmvNOWY1-W1aOAwYqNDkDDmx-5UVlCiXAtxUZ8A,1057
 vec_inf/cli/_vars.py,sha256=V6DrJs_BuUa4yNcbBSSnMwpcyXwEBsizy3D0ubIg2fA,777
 vec_inf/client/__init__.py,sha256=OLlUJ4kL1R-Kh-nXNbvKlAZ3mtHcnozHprVufkVCNWk,739
-vec_inf/client/_client_vars.py,sha256=eVQjpuASd8beBjAeAbQnMRZM8nCLZMHx-x62BcXVnYA,7163
+vec_inf/client/_client_vars.py,sha256=KG-xImVIzJH3aj5nMUzT9w9LpH-7YGrOew6N77Fj0Js,7638
 vec_inf/client/_exceptions.py,sha256=94Nx_5k1SriJNXzbdnwyXFZolyMutydU08Gsikawzzo,749
-vec_inf/client/_helper.py,sha256=76OTCroNR5e3e7T2qSV_tkexDaUQsJrs8bFiMJ5NaxU,22718
-vec_inf/client/_slurm_script_generator.py,sha256=jFgr2Pu7b_Uqli3DBvxUr9MI1-3TA6wwxg07O2rTwPs,6299
+vec_inf/client/_helper.py,sha256=DcEFogbrSb4A8Kc2zixNZNL4nt4iswPk2n5blZgwEWQ,22338
+vec_inf/client/_slurm_script_generator.py,sha256=XYCsadCLDEu9KrrjrNCNgoc0ITmjys9u7yWR9PkFAos,6376
 vec_inf/client/_utils.py,sha256=1dB2O1neEhZNk6MJbBybLQm42vsmEevA2TI0F_kGi0o,8796
 vec_inf/client/api.py,sha256=TYn4lP5Ene8MEuXWYo6ZbGYw9aPnaMlT32SH7jLCifM,9605
-vec_inf/client/config.py,sha256=kOhxoepsvArxRFNlwq1sLDHsxDewLwxRV1VwsL0MrGU,4683
-vec_inf/client/models.py,sha256=JZDUMBX3XKOClaq_yJUpDUSgiDy42nT5Dq5bxQWiO2I,5778
+vec_inf/client/config.py,sha256=lPVHwiaGZjKd5M9G7vcsk3DMausFP_telq3JQngBkH8,5080
+vec_inf/client/models.py,sha256=qjocUa5egJTVeVF3962kYOecs1dTaEb2e6TswkYFXM0,6141
 vec_inf/client/slurm_vars.py,sha256=lroK41L4gEVVZNxxE3bEpbKsdMwnH79-7iCKd4zWEa4,1069
 vec_inf/config/README.md,sha256=OlgnD_Ojei_xLkNyS7dGvYMFUzQFqjVRVw0V-QMk_3g,17863
-vec_inf/config/models.yaml,sha256=PR91vOzINVOkAco9S-_VIXQ5Un6ekeoWz2Pj8DMR8LQ,29630
-vec_inf-0.6.0.dist-info/METADATA,sha256=-xadTsrAR3tOfPyxTdGB9DLuhWMu_mnp_JF5Aa-1-08,9755
-vec_inf-0.6.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-vec_inf-0.6.0.dist-info/entry_points.txt,sha256=uNRXjCuJSR2nveEqD3IeMznI9oVI9YLZh5a24cZg6B0,49
-vec_inf-0.6.0.dist-info/licenses/LICENSE,sha256=mq8zeqpvVSF1EsxmydeXcokt8XnEIfSofYn66S2-cJI,1073
-vec_inf-0.6.0.dist-info/RECORD,,
+vec_inf/config/models.yaml,sha256=xImSOjG9yL6LqqYkSLL7_wBZhqKM10-eFaQJ82gP4ig,29420
+vec_inf-0.6.1.dist-info/METADATA,sha256=0YHT8rhEZINfmMF1hQBqU0HBpRbwX-1IeqY_Mla4g28,10682
+vec_inf-0.6.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+vec_inf-0.6.1.dist-info/entry_points.txt,sha256=uNRXjCuJSR2nveEqD3IeMznI9oVI9YLZh5a24cZg6B0,49
+vec_inf-0.6.1.dist-info/licenses/LICENSE,sha256=mq8zeqpvVSF1EsxmydeXcokt8XnEIfSofYn66S2-cJI,1073
+vec_inf-0.6.1.dist-info/RECORD,,