vec-inf 0.4.1__py3-none-any.whl → 0.5.0__py3-none-any.whl

vec_inf/multinode_vllm.slurm

@@ -4,15 +4,10 @@
 #SBATCH --exclusive
 #SBATCH --tasks-per-node=1
 
-# Load CUDA, change to the cuda version on your environment if different
-source /opt/lmod/lmod/init/profile
-module load cuda-12.3
-nvidia-smi
-
 source ${SRC_DIR}/find_port.sh
 
 if [ "$VENV_BASE" = "singularity" ]; then
-    export SINGULARITY_IMAGE=/projects/aieng/public/vector-inference_latest.sif
+    export SINGULARITY_IMAGE=/model-weights/vec-inf-shared/vector-inference_latest.sif
     export VLLM_NCCL_SO_PATH=/vec-inf/nccl/libnccl.so.2.18.1
     module load singularity-ce/3.8.2
     singularity exec $SINGULARITY_IMAGE ray stop
@@ -36,13 +31,13 @@ echo "IP Head: $ip_head"
 echo "Starting HEAD at $head_node"
 if [ "$VENV_BASE" = "singularity" ]; then
     srun --nodes=1 --ntasks=1 -w "$head_node" \
-        singularity exec --nv --bind ${MODEL_WEIGHTS_PARENT_DIR}:${MODEL_WEIGHTS_PARENT_DIR} $SINGULARITY_IMAGE \
+        singularity exec --nv --bind ${MODEL_WEIGHTS}:${MODEL_WEIGHTS} $SINGULARITY_IMAGE \
         ray start --head --node-ip-address="$head_node_ip" --port=$head_node_port \
-        --num-cpus "${SLURM_CPUS_PER_TASK}" --num-gpus "${NUM_GPUS}" --block &
+        --num-cpus "${SLURM_CPUS_PER_TASK}" --num-gpus "${SLURM_GPUS_PER_NODE}" --block &
 else
     srun --nodes=1 --ntasks=1 -w "$head_node" \
         ray start --head --node-ip-address="$head_node_ip" --port=$head_node_port \
-        --num-cpus "${SLURM_CPUS_PER_TASK}" --num-gpus "${NUM_GPUS}" --block &
+        --num-cpus "${SLURM_CPUS_PER_TASK}" --num-gpus "${SLURM_GPUS_PER_NODE}" --block &
 fi
 
 # Starting the Ray worker nodes
@@ -57,13 +52,13 @@ for ((i = 1; i <= worker_num; i++)); do
     echo "Starting WORKER $i at $node_i"
     if [ "$VENV_BASE" = "singularity" ]; then
         srun --nodes=1 --ntasks=1 -w "$node_i" \
-            singularity exec --nv --bind ${MODEL_WEIGHTS_PARENT_DIR}:${MODEL_WEIGHTS_PARENT_DIR} $SINGULARITY_IMAGE \
+            singularity exec --nv --bind ${MODEL_WEIGHTS}:${MODEL_WEIGHTS} $SINGULARITY_IMAGE \
             ray start --address "$ip_head" \
-            --num-cpus "${SLURM_CPUS_PER_TASK}" --num-gpus "${NUM_GPUS}" --block &
+            --num-cpus "${SLURM_CPUS_PER_TASK}" --num-gpus "${SLURM_GPUS_PER_NODE}" --block &
     else
         srun --nodes=1 --ntasks=1 -w "$node_i" \
             ray start --address "$ip_head" \
-            --num-cpus "${SLURM_CPUS_PER_TASK}" --num-gpus "${NUM_GPUS}" --block &
+            --num-cpus "${SLURM_CPUS_PER_TASK}" --num-gpus "${SLURM_GPUS_PER_NODE}" --block &
     fi
 
     sleep 5
@@ -72,14 +67,21 @@ done
 
 vllm_port_number=$(find_available_port $head_node_ip 8080 65535)
 
-echo "Server address: http://${head_node_ip}:${vllm_port_number}/v1"
+SERVER_ADDR="http://${head_node_ip}:${vllm_port_number}/v1"
+echo "Server address: $SERVER_ADDR"
+
+jq --arg server_addr "$SERVER_ADDR" \
+  '. + {"server_address": $server_addr}' \
+  "$LOG_DIR/$MODEL_NAME.$SLURM_JOB_ID/$MODEL_NAME.$SLURM_JOB_ID.json" > temp.json \
+  && mv temp.json "$LOG_DIR/$MODEL_NAME.$SLURM_JOB_ID/$MODEL_NAME.$SLURM_JOB_ID.json" \
+  && rm temp.json
 
 if [ "$PIPELINE_PARALLELISM" = "True" ]; then
-    export PIPELINE_PARALLEL_SIZE=$NUM_NODES
-    export TENSOR_PARALLEL_SIZE=$NUM_GPUS
+    export PIPELINE_PARALLEL_SIZE=$SLURM_JOB_NUM_NODES
+    export TENSOR_PARALLEL_SIZE=$SLURM_GPUS_PER_NODE
 else
     export PIPELINE_PARALLEL_SIZE=1
-    export TENSOR_PARALLEL_SIZE=$((NUM_NODES*NUM_GPUS))
+    export TENSOR_PARALLEL_SIZE=$((SLURM_JOB_NUM_NODES*SLURM_GPUS_PER_NODE))
 fi
 
 if [ "$ENFORCE_EAGER" = "True" ]; then
@@ -88,37 +90,65 @@ else
     export ENFORCE_EAGER=""
 fi
 
+if [ "$ENABLE_PREFIX_CACHING" = "True" ]; then
+    export ENABLE_PREFIX_CACHING="--enable-prefix-caching"
+else
+    export ENABLE_PREFIX_CACHING=""
+fi
+
+if [ "$ENABLE_CHUNKED_PREFILL" = "True" ]; then
+    export ENABLE_CHUNKED_PREFILL="--enable-chunked-prefill"
+else
+    export ENABLE_CHUNKED_PREFILL=""
+fi
+
+if [ -z "$MAX_NUM_BATCHED_TOKENS" ]; then
+    export MAX_NUM_BATCHED_TOKENS=""
+else
+    export MAX_NUM_BATCHED_TOKENS="--max-num-batched-tokens=$MAX_NUM_BATCHED_TOKENS"
+fi
+
 # Activate vllm venv
 if [ "$VENV_BASE" = "singularity" ]; then
-    singularity exec --nv --bind ${MODEL_WEIGHTS_PARENT_DIR}:${MODEL_WEIGHTS_PARENT_DIR} $SINGULARITY_IMAGE \
+    singularity exec --nv --bind ${MODEL_WEIGHTS}:${MODEL_WEIGHTS} $SINGULARITY_IMAGE \
     python3.10 -m vllm.entrypoints.openai.api_server \
-    --model ${VLLM_MODEL_WEIGHTS} \
-    --served-model-name ${JOB_NAME} \
+    --model ${MODEL_WEIGHTS} \
+    --served-model-name ${MODEL_NAME} \
     --host "0.0.0.0" \
     --port ${vllm_port_number} \
     --pipeline-parallel-size ${PIPELINE_PARALLEL_SIZE} \
     --tensor-parallel-size ${TENSOR_PARALLEL_SIZE} \
-    --dtype ${VLLM_DATA_TYPE} \
+    --dtype ${DATA_TYPE} \
     --trust-remote-code \
-    --max-logprobs ${VLLM_MAX_LOGPROBS} \
-    --max-model-len ${VLLM_MAX_MODEL_LEN} \
-    --max-num-seqs ${VLLM_MAX_NUM_SEQS} \
-    --task ${VLLM_TASK} \
+    --max-logprobs ${MAX_LOGPROBS} \
+    --max-model-len ${MAX_MODEL_LEN} \
+    --max-num-seqs ${MAX_NUM_SEQS} \
+    --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \
+    --compilation-config ${COMPILATION_CONFIG} \
+    --task ${TASK} \
+    ${MAX_NUM_BATCHED_TOKENS} \
+    ${ENABLE_PREFIX_CACHING} \
+    ${ENABLE_CHUNKED_PREFILL} \
     ${ENFORCE_EAGER}
 else
     source ${VENV_BASE}/bin/activate
     python3 -m vllm.entrypoints.openai.api_server \
-    --model ${VLLM_MODEL_WEIGHTS} \
-    --served-model-name ${JOB_NAME} \
+    --model ${MODEL_WEIGHTS} \
+    --served-model-name ${MODEL_NAME} \
     --host "0.0.0.0" \
     --port ${vllm_port_number} \
     --pipeline-parallel-size ${PIPELINE_PARALLEL_SIZE} \
     --tensor-parallel-size ${TENSOR_PARALLEL_SIZE} \
-    --dtype ${VLLM_DATA_TYPE} \
+    --dtype ${DATA_TYPE} \
     --trust-remote-code \
-    --max-logprobs ${VLLM_MAX_LOGPROBS} \
-    --max-model-len ${VLLM_MAX_MODEL_LEN} \
-    --max-num-seqs ${VLLM_MAX_NUM_SEQS} \
-    --task ${VLLM_TASK} \
+    --max-logprobs ${MAX_LOGPROBS} \
+    --max-model-len ${MAX_MODEL_LEN} \
+    --max-num-seqs ${MAX_NUM_SEQS} \
+    --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \
+    --compilation-config ${COMPILATION_CONFIG} \
+    --task ${TASK} \
+    ${MAX_NUM_BATCHED_TOKENS} \
+    ${ENABLE_PREFIX_CACHING} \
+    ${ENABLE_CHUNKED_PREFILL} \
     ${ENFORCE_EAGER}
 fi

vec_inf/vllm.slurm

@@ -2,18 +2,20 @@
 #SBATCH --cpus-per-task=16
 #SBATCH --mem=64G
 
-# Load CUDA, change to the cuda version on your environment if different
-source /opt/lmod/lmod/init/profile
-module load cuda-12.3
-nvidia-smi
-
 source ${SRC_DIR}/find_port.sh
 
 # Write server url to file
 hostname=${SLURMD_NODENAME}
 vllm_port_number=$(find_available_port $hostname 8080 65535)
 
-echo "Server address: http://${hostname}:${vllm_port_number}/v1"
+SERVER_ADDR="http://${hostname}:${vllm_port_number}/v1"
+echo "Server address: $SERVER_ADDR"
+
+jq --arg server_addr "$SERVER_ADDR" \
+  '. + {"server_address": $server_addr}' \
+  "$LOG_DIR/$MODEL_NAME.$SLURM_JOB_ID/$MODEL_NAME.$SLURM_JOB_ID.json" > temp.json \
+  && mv temp.json "$LOG_DIR/$MODEL_NAME.$SLURM_JOB_ID/$MODEL_NAME.$SLURM_JOB_ID.json" \
+  && rm temp.json
 
 if [ "$ENFORCE_EAGER" = "True" ]; then
     export ENFORCE_EAGER="--enforce-eager"
@@ -21,39 +23,68 @@ else
     export ENFORCE_EAGER=""
 fi
 
+if [ "$ENABLE_PREFIX_CACHING" = "True" ]; then
+    export ENABLE_PREFIX_CACHING="--enable-prefix-caching"
+else
+    export ENABLE_PREFIX_CACHING=""
+fi
+
+if [ "$ENABLE_CHUNKED_PREFILL" = "True" ]; then
+    export ENABLE_CHUNKED_PREFILL="--enable-chunked-prefill"
+else
+    export ENABLE_CHUNKED_PREFILL=""
+fi
+
+if [ -z "$MAX_NUM_BATCHED_TOKENS" ]; then
+    export MAX_NUM_BATCHED_TOKENS=""
+else
+    export MAX_NUM_BATCHED_TOKENS="--max-num-batched-tokens=$MAX_NUM_BATCHED_TOKENS"
+fi
+
 # Activate vllm venv
 if [ "$VENV_BASE" = "singularity" ]; then
-    export SINGULARITY_IMAGE=/projects/aieng/public/vector-inference_latest.sif
+    export SINGULARITY_IMAGE=/model-weights/vec-inf-shared/vector-inference_latest.sif
     export VLLM_NCCL_SO_PATH=/vec-inf/nccl/libnccl.so.2.18.1
     module load singularity-ce/3.8.2
     singularity exec $SINGULARITY_IMAGE ray stop
-    singularity exec --nv --bind ${MODEL_WEIGHTS_PARENT_DIR}:${MODEL_WEIGHTS_PARENT_DIR} $SINGULARITY_IMAGE \
+    singularity exec --nv --bind ${MODEL_WEIGHTS}:${MODEL_WEIGHTS} $SINGULARITY_IMAGE \
     python3.10 -m vllm.entrypoints.openai.api_server \
-    --model ${VLLM_MODEL_WEIGHTS} \
-    --served-model-name ${JOB_NAME} \
+    --model ${MODEL_WEIGHTS} \
+    --served-model-name ${MODEL_NAME} \
     --host "0.0.0.0" \
     --port ${vllm_port_number} \
-    --tensor-parallel-size ${NUM_GPUS} \
-    --dtype ${VLLM_DATA_TYPE} \
-    --max-logprobs ${VLLM_MAX_LOGPROBS} \
+    --tensor-parallel-size ${SLURM_GPUS_PER_NODE} \
+    --dtype ${DATA_TYPE} \
+    --max-logprobs ${MAX_LOGPROBS} \
     --trust-remote-code \
-    --max-model-len ${VLLM_MAX_MODEL_LEN} \
-    --max-num-seqs ${VLLM_MAX_NUM_SEQS} \
-    --task ${VLLM_TASK} \
+    --max-model-len ${MAX_MODEL_LEN} \
+    --max-num-seqs ${MAX_NUM_SEQS} \
+    --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \
+    --compilation-config ${COMPILATION_CONFIG} \
+    --task ${TASK} \
+    ${MAX_NUM_BATCHED_TOKENS} \
+    ${ENABLE_PREFIX_CACHING} \
+    ${ENABLE_CHUNKED_PREFILL} \
     ${ENFORCE_EAGER}
+
 else
     source ${VENV_BASE}/bin/activate
     python3 -m vllm.entrypoints.openai.api_server \
-    --model ${VLLM_MODEL_WEIGHTS} \
-    --served-model-name ${JOB_NAME} \
+    --model ${MODEL_WEIGHTS} \
+    --served-model-name ${MODEL_NAME} \
     --host "0.0.0.0" \
     --port ${vllm_port_number} \
-    --tensor-parallel-size ${NUM_GPUS} \
-    --dtype ${VLLM_DATA_TYPE} \
-    --max-logprobs ${VLLM_MAX_LOGPROBS} \
+    --tensor-parallel-size ${SLURM_GPUS_PER_NODE} \
+    --dtype ${DATA_TYPE} \
+    --max-logprobs ${MAX_LOGPROBS} \
     --trust-remote-code \
-    --max-model-len ${VLLM_MAX_MODEL_LEN} \
-    --max-num-seqs ${VLLM_MAX_NUM_SEQS} \
-    --task ${VLLM_TASK} \
+    --max-model-len ${MAX_MODEL_LEN} \
+    --max-num-seqs ${MAX_NUM_SEQS} \
+    --gpu-memory-utilization ${GPU_MEMORY_UTILIZATION} \
+    --compilation-config ${COMPILATION_CONFIG} \
+    --task ${TASK} \
+    ${MAX_NUM_BATCHED_TOKENS} \
+    ${ENABLE_PREFIX_CACHING} \
+    ${ENABLE_CHUNKED_PREFILL} \
     ${ENFORCE_EAGER}
 fi

vec_inf-0.5.0.dist-info/METADATA

@@ -0,0 +1,210 @@
+Metadata-Version: 2.4
+Name: vec-inf
+Version: 0.5.0
+Summary: Efficient LLM inference on Slurm clusters using vLLM.
+Author-email: Marshall Wang <marshall.wang@vectorinstitute.ai>
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.10
+Requires-Dist: click>=8.1.0
+Requires-Dist: pydantic>=2.10.6
+Requires-Dist: pyyaml>=6.0.2
+Requires-Dist: requests>=2.31.0
+Requires-Dist: rich>=13.7.0
+Provides-Extra: dev
+Requires-Dist: cupy-cuda12x==12.1.0; extra == 'dev'
+Requires-Dist: ray>=2.40.0; extra == 'dev'
+Requires-Dist: torch>=2.5.1; extra == 'dev'
+Requires-Dist: vllm-nccl-cu12<2.19,>=2.18; extra == 'dev'
+Requires-Dist: vllm>=0.7.3; extra == 'dev'
+Requires-Dist: xgrammar>=0.1.11; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# Vector Inference: Easy inference on Slurm clusters
+
+----------------------------------------------------
+
+[![PyPI](https://img.shields.io/pypi/v/vec-inf)](https://pypi.org/project/vec-inf)
+[![code checks](https://github.com/VectorInstitute/vector-inference/actions/workflows/code_checks.yml/badge.svg)](https://github.com/VectorInstitute/vector-inference/actions/workflows/code_checks.yml)
+[![docs](https://github.com/VectorInstitute/vector-inference/actions/workflows/docs_deploy.yml/badge.svg)](https://github.com/VectorInstitute/vector-inference/actions/workflows/docs_deploy.yml)
+[![codecov](https://codecov.io/github/VectorInstitute/vector-inference/branch/develop/graph/badge.svg?token=NI88QSIGAC)](https://app.codecov.io/github/VectorInstitute/vector-inference/tree/develop)
+![GitHub License](https://img.shields.io/github/license/VectorInstitute/vector-inference)
+
+This repository provides an easy-to-use solution to run inference servers on [Slurm](https://slurm.schedmd.com/overview.html)-managed computing clusters using [vLLM](https://docs.vllm.ai/en/latest/). **All scripts in this repository runs natively on the Vector Institute cluster environment**. To adapt to other environments, update the environment variables in [`cli/_helper.py`](vec_inf/cli/_helper.py), [`cli/_config.py`](vec_inf/cli/_config.py), [`vllm.slurm`](vec_inf/vllm.slurm), [`multinode_vllm.slurm`](vec_inf/multinode_vllm.slurm) and [`models.yaml`](vec_inf/config/models.yaml) accordingly.
+
+## Installation
+If you are using the Vector cluster environment, and you don't need any customization to the inference server environment, run the following to install package:
+
+```bash
+pip install vec-inf
+```
+Otherwise, we recommend using the provided [`Dockerfile`](Dockerfile) to set up your own environment with the package
+
+## Usage
+
+### `launch` command
+
+The `launch` command allows users to deploy a model as a slurm job. If the job successfully launches, a URL endpoint is exposed for the user to send requests for inference.
+
+We will use the Llama 3.1 model as example, to launch an OpenAI compatible inference server for Meta-Llama-3.1-8B-Instruct, run:
+
+```bash
+vec-inf launch Meta-Llama-3.1-8B-Instruct
+```
+You should see an output like the following:
+
+<img width="600" alt="launch_img" src="https://github.com/user-attachments/assets/883e6a5b-8016-4837-8fdf-39097dfb18bf">
+
+
+#### Overrides
+
+Models that are already supported by `vec-inf` would be launched using the cached configuration or [default configuration](vec_inf/config/models.yaml). You can override these values by providing additional parameters. Use `vec-inf launch --help` to see the full list of parameters that can be
+overriden. For example, if `qos` is to be overriden:
+
+```bash
+vec-inf launch Meta-Llama-3.1-8B-Instruct --qos <new_qos>
+```
+
+#### Custom models
+
+You can also launch your own custom model as long as the model architecture is [supported by vLLM](https://docs.vllm.ai/en/stable/models/supported_models.html), and make sure to follow the instructions below:
+* Your model weights directory naming convention should follow `$MODEL_FAMILY-$MODEL_VARIANT` ($MODEL_VARIANT is OPTIONAL).
+* Your model weights directory should contain HuggingFace format weights.
+* You should specify your model configuration by:
+  * Creating a custom configuration file for your model and specify its path via setting the environment variable `VEC_INF_CONFIG`. Check the [default parameters](vec_inf/config/models.yaml) file for the format of the config file. All the parameters for the model should be specified in that config file.
+  * Using launch command options to specify your model setup.
+* For other model launch parameters you can reference the default values for similar models using the [`list` command ](#list-command).
+
+Here is an example to deploy a custom [Qwen2.5-7B-Instruct-1M](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct-1M) model which is not
+supported in the default list of models using a user custom config. In this case, the model weights are assumed to be downloaded to
+a `model-weights` directory inside the user's home directory. The weights directory of the model follows the naming convention so it
+would be named `Qwen2.5-7B-Instruct-1M`. The following yaml file would need to be created, lets say it is named `/h/<username>/my-model-config.yaml`.
+
+```yaml
+models:
+  Qwen2.5-7B-Instruct-1M:
+    model_family: Qwen2.5
+    model_variant: 7B-Instruct-1M
+    model_type: LLM
+    gpus_per_node: 1
+    num_nodes: 1
+    vocab_size: 152064
+    max_model_len: 1010000
+    max_num_seqs: 256
+    pipeline_parallelism: true
+    enforce_eager: false
+    qos: m2
+    time: 08:00:00
+    partition: a40
+    model_weights_parent_dir: /h/<username>/model-weights
+```
+
+You would then set the `VEC_INF_CONFIG` path using:
+
+```bash
+export VEC_INF_CONFIG=/h/<username>/my-model-config.yaml
+```
+
+Note that there are other parameters that can also be added to the config but not shown in this example, such as `data_type` and `log_dir`.
+
+### `status` command
+You can check the inference server status by providing the Slurm job ID to the `status` command:
+```bash
+vec-inf status 15373800
+```
+
+If the server is pending for resources, you should see an output like this:
+
+<img width="400" alt="status_pending_img" src="https://github.com/user-attachments/assets/b659c302-eae1-4560-b7a9-14eb3a822a2f">
+
+When the server is ready, you should see an output like this:
+
+<img width="400" alt="status_ready_img" src="https://github.com/user-attachments/assets/672986c2-736c-41ce-ac7c-1fb585cdcb0d">
+
+There are 5 possible states:
+
+* **PENDING**: Job submitted to Slurm, but not executed yet. Job pending reason will be shown.
+* **LAUNCHING**: Job is running but the server is not ready yet.
+* **READY**: Inference server running and ready to take requests.
+* **FAILED**: Inference server in an unhealthy state. Job failed reason will be shown.
+* **SHUTDOWN**: Inference server is shutdown/cancelled.
+
+Note that the base URL is only available when model is in `READY` state, and if you've changed the Slurm log directory path, you also need to specify it when using the `status` command.
+
+### `metrics` command
+Once your server is ready, you can check performance metrics by providing the Slurm job ID to the `metrics` command:
+```bash
+vec-inf metrics 15373800
+```
+
+And you will see the performance metrics streamed to your console, note that the metrics are updated with a 2-second interval.
+
+<img width="400" alt="metrics_img" src="https://github.com/user-attachments/assets/3ee143d0-1a71-4944-bbd7-4c3299bf0339">
+
+### `shutdown` command
+Finally, when you're finished using a model, you can shut it down by providing the Slurm job ID:
+```bash
+vec-inf shutdown 15373800
+
+> Shutting down model with Slurm Job ID: 15373800
+```
+
+### `list` command
+You call view the full list of available models by running the `list` command:
+```bash
+vec-inf list
+```
+<img width="940" alt="list_img" src="https://github.com/user-attachments/assets/8cf901c4-404c-4398-a52f-0486f00747a3">
+
+NOTE: The above screenshot does not represent the full list of models supported.
+
+You can also view the default setup for a specific supported model by providing the model name, for example `Meta-Llama-3.1-70B-Instruct`:
+```bash
+vec-inf list Meta-Llama-3.1-70B-Instruct
+```
+<img width="500" alt="list_model_img" src="https://github.com/user-attachments/assets/34e53937-2d86-443e-85f6-34e408653ddb">
+
+`launch`, `list`, and `status` command supports `--json-mode`, where the command output would be structured as a JSON string.
+
+## Send inference requests
+Once the inference server is ready, you can start sending in inference requests. We provide example scripts for sending inference requests in [`examples`](examples) folder. Make sure to update the model server URL and the model weights location in the scripts. For example, you can run `python examples/inference/llm/chat_completions.py`, and you should expect to see an output like the following:
+
+```json
+{
+    "id":"chatcmpl-387c2579231948ffaf66cdda5439d3dc",
+    "choices": [
+        {
+            "finish_reason":"stop",
+            "index":0,
+            "logprobs":null,
+            "message": {
+                "content":"Arrr, I be Captain Chatbeard, the scurviest chatbot on the seven seas! Ye be wantin' to know me identity, eh? Well, matey, I be a swashbucklin' AI, here to provide ye with answers and swappin' tales, savvy?",
+                "role":"assistant",
+                "function_call":null,
+                "tool_calls":[],
+                "reasoning_content":null
+            },
+            "stop_reason":null
+        }
+    ],
+    "created":1742496683,
+    "model":"Meta-Llama-3.1-8B-Instruct",
+    "object":"chat.completion",
+    "system_fingerprint":null,
+    "usage": {
+        "completion_tokens":66,
+        "prompt_tokens":32,
+        "total_tokens":98,
+        "prompt_tokens_details":null
+    },
+    "prompt_logprobs":null
+}
+```
+**NOTE**: For multimodal models, currently only `ChatCompletion` is available, and only one image can be provided for each prompt.
+
+## SSH tunnel from your local device
+If you want to run inference from your local device, you can open a SSH tunnel to your cluster environment like the following:
+```bash
+ssh -L 8081:172.17.8.29:8081 username@v.vectorinstitute.ai -N
+```
+Where the last number in the URL is the GPU number (gpu029 in this case). The example provided above is for the vector cluster, change the variables accordingly for your environment

vec_inf-0.5.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+vec_inf/README.md,sha256=dxX0xKfwLioG0mJ2YFv5JJ5q1m5NlWBrVBOap1wuHfQ,624
+vec_inf/__init__.py,sha256=bHwSIz9lebYuxIemni-lP0h3gwJHVbJnwExQKGJWw_Q,23
+vec_inf/find_port.sh,sha256=bGQ6LYSFVSsfDIGatrSg5YvddbZfaPL0R-Bjo4KYD6I,1088
+vec_inf/multinode_vllm.slurm,sha256=V01ayfgObPdxbQqhYvCbNIx0zqpLurDxZhS0UHYNFi0,5210
+vec_inf/vllm.slurm,sha256=VMMTdVUOtX4-Yv43yzgKiEpE56fMwuR0KOLf3Dar_S0,2884
+vec_inf/cli/__init__.py,sha256=5XIvGQCOnaGl73XMkwetjC-Ul3xuXGrWDXdYJ3aUzvU,27
+vec_inf/cli/_cli.py,sha256=8Gk4NRbrY2-3EX0S8_-1UOmGfahzqX0A2sJNVcL7OL8,6525
+vec_inf/cli/_config.py,sha256=pb2ERbxZoRZBa9Ie7-jlzyQiiXYZgqUetbw13Blryho,2841
+vec_inf/cli/_helper.py,sha256=9niBuFoaJfeP2yRHKcrkia4rwCZbAfoz-4s2MVCwA9w,26871
+vec_inf/cli/_utils.py,sha256=i3mffIJ-wBVpe3pz0mYa2W5J42yNRmUG2tQwABhQJDQ,5365
+vec_inf/config/README.md,sha256=3MYYY3hGw7jiMR5A8CMdBkhGFobcdH3Kip5E2saq_T4,18609
+vec_inf/config/models.yaml,sha256=p__omZEmoF93BtVQXQia43xvPX88ALrBT1tMiY2Bdhk,28787
+vec_inf-0.5.0.dist-info/METADATA,sha256=khBhIsW5hFjrp-ZQFU3wa5htLVkp4CH3JnrhLXeHfss,10228
+vec_inf-0.5.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+vec_inf-0.5.0.dist-info/entry_points.txt,sha256=uNRXjCuJSR2nveEqD3IeMznI9oVI9YLZh5a24cZg6B0,49
+vec_inf-0.5.0.dist-info/licenses/LICENSE,sha256=mq8zeqpvVSF1EsxmydeXcokt8XnEIfSofYn66S2-cJI,1073
+vec_inf-0.5.0.dist-info/RECORD,,

vec_inf/launch_server.sh

@@ -1,145 +0,0 @@
-#!/bin/bash
-
-# ================================= Read Named Args ======================================
-
-while [[ "$#" -gt 0 ]]; do
-    case $1 in
-        --model-family) model_family="$2"; shift ;;
-        --model-variant) model_variant="$2"; shift ;;
-        --model-type) model_type="$2"; shift ;;
-        --partition) partition="$2"; shift ;;
-        --qos) qos="$2"; shift ;;
-        --time) walltime="$2"; shift ;;
-        --num-nodes) num_nodes="$2"; shift ;;
-        --num-gpus) num_gpus="$2"; shift ;;
-        --max-model-len) max_model_len="$2"; shift ;;
-        --max-num-seqs) max_num_seqs="$2"; shift ;;
-        --vocab-size) vocab_size="$2"; shift ;;
-        --data-type) data_type="$2"; shift ;;
-        --venv) venv="$2"; shift ;;
-        --log-dir) log_dir="$2"; shift ;;
-        --model-weights-parent-dir) model_weights_parent_dir="$2"; shift ;;
-        --pipeline-parallelism) pipeline_parallelism="$2"; shift ;;
-        --enforce-eager) enforce_eager="$2"; shift ;;
-        *) echo "Unknown parameter passed: $1"; exit 1 ;;
-    esac
-    shift
-done
-
-required_vars=(model_family model_variant model_type partition qos walltime num_nodes num_gpus max_model_len vocab_size data_type venv log_dir model_weights_parent_dir)
-
-for var in "$required_vars[@]"; do
-    if [ -z "$!var" ]; then
-        echo "Error: Missing required --$var argument."
-        exit 1
-    fi
-done
-
-export MODEL_FAMILY=$model_family
-export MODEL_VARIANT=$model_variant
-export MODEL_TYPE=$model_type
-export JOB_PARTITION=$partition
-export QOS=$qos
-export WALLTIME=$walltime
-export NUM_NODES=$num_nodes
-export NUM_GPUS=$num_gpus
-export VLLM_MAX_MODEL_LEN=$max_model_len
-export VLLM_MAX_LOGPROBS=$vocab_size
-export VLLM_DATA_TYPE=$data_type
-export VENV_BASE=$venv
-export LOG_DIR=$log_dir
-export MODEL_WEIGHTS_PARENT_DIR=$model_weights_parent_dir
-
-if [[ "$model_type" == "LLM" || "$model_type" == "VLM" ]]; then
-    export VLLM_TASK="generate"
-elif [ "$model_type" == "Reward_Modeling" ]; then
-    export VLLM_TASK="reward"
-elif [ "$model_type" == "Text_Embedding" ]; then
-    export VLLM_TASK="embed"
-else
-    echo "Error: Unknown model_type: $model_type"
-    exit 1
-fi
-
-if [ -n "$max_num_seqs" ]; then
-    export VLLM_MAX_NUM_SEQS=$max_num_seqs
-else
-    export VLLM_MAX_NUM_SEQS=256
-fi
-
-if [ -n "$pipeline_parallelism" ]; then
-    export PIPELINE_PARALLELISM=$pipeline_parallelism
-else
-    export PIPELINE_PARALLELISM="False"
-fi
-
-if [ -n "$enforce_eager" ]; then
-    export ENFORCE_EAGER=$enforce_eager
-else
-    export ENFORCE_EAGER="False"
-fi
-
-# ================================= Set default environment variables ======================================
-# Slurm job configuration
-export JOB_NAME="$MODEL_FAMILY-$MODEL_VARIANT"
-if [ "$JOB_NAME" == "DeepSeek-R1-None" ]; then
-    export JOB_NAME=$MODEL_FAMILY
-fi
-
-if [ "$LOG_DIR" = "default" ]; then
-    export LOG_DIR="$HOME/.vec-inf-logs/$MODEL_FAMILY"
-fi
-mkdir -p $LOG_DIR
-
-# Model and entrypoint configuration. API Server URL (host, port) are set automatically based on the
-# SLURM job
-export SRC_DIR="$(dirname "$0")"
-export MODEL_DIR="${SRC_DIR}/models/${MODEL_FAMILY}"
-
-# Variables specific to your working environment, below are examples for the Vector cluster
-export VLLM_MODEL_WEIGHTS="${MODEL_WEIGHTS_PARENT_DIR}/${JOB_NAME}"
-export LD_LIBRARY_PATH="/scratch/ssd001/pkgs/cudnn-11.7-v8.5.0.96/lib/:/scratch/ssd001/pkgs/cuda-11.7/targets/x86_64-linux/lib/"
-
-
-# ================================ Validate Inputs & Launch Server =================================
-
-# Set data type to fp16 instead of bf16 for non-Ampere GPUs
-fp16_partitions="t4v1 t4v2"
-
-# choose from 'auto', 'half', 'float16', 'bfloat16', 'float', 'float32'
-if [[ $fp16_partitions =~ $JOB_PARTITION ]]; then
-    export VLLM_DATA_TYPE="float16"
-    echo "Data type set to due to non-Ampere GPUs used: $VLLM_DATA_TYPE"
-fi
-
-echo Job Name: $JOB_NAME
-echo Partition: $JOB_PARTITION
-echo Num Nodes: $NUM_NODES
-echo GPUs per Node: $NUM_GPUS
-echo QOS: $QOS
-echo Walltime: $WALLTIME
-echo Model Type: $MODEL_TYPE
-echo Task: $VLLM_TASK
-echo Data Type: $VLLM_DATA_TYPE
-echo Max Model Length: $VLLM_MAX_MODEL_LEN
-echo Max Num Seqs: $VLLM_MAX_NUM_SEQS
-echo Vocabulary Size: $VLLM_MAX_LOGPROBS
-echo Pipeline Parallelism: $PIPELINE_PARALLELISM
-echo Enforce Eager: $ENFORCE_EAGER
-echo Log Directory: $LOG_DIR
-echo Model Weights Parent Directory: $MODEL_WEIGHTS_PARENT_DIR
-
-is_special=""
-if [ "$NUM_NODES" -gt 1 ]; then
-    is_special="multinode_"
-fi
-
-sbatch --job-name $JOB_NAME \
-    --partition $JOB_PARTITION \
-    --nodes $NUM_NODES \
-    --gres gpu:$NUM_GPUS \
-    --qos $QOS \
-    --time $WALLTIME \
-    --output $LOG_DIR/$JOB_NAME.%j.out \
-    --error $LOG_DIR/$JOB_NAME.%j.err \
-    $SRC_DIR/${is_special}vllm.slurm