vec-inf 0.7.0__py3-none-any.whl → 0.7.1__py3-none-any.whl

vec_inf/client/_client_vars.py

@@ -71,10 +71,3 @@ VLLM_SHORT_TO_LONG_MAP = {
 
 # Required matching arguments for batch mode
 BATCH_MODE_REQUIRED_MATCHING_ARGS = ["venv", "log_dir"]
-
-# Required arguments for launching jobs that don't have a default value and their
-# corresponding environment variables
-REQUIRED_ARGS = {
-    "account": "VEC_INF_ACCOUNT",
-    "work_dir": "VEC_INF_WORK_DIR",
-}

vec_inf/client/_slurm_vars.py

@@ -78,5 +78,9 @@ RESOURCE_TYPE: TypeAlias = create_literal_type(  # type: ignore[valid-type]
     _config["allowed_values"]["resource_type"]
 )
 
+# Extract required arguments, for launching jobs that don't have a default value and
+# their corresponding environment variables
+REQUIRED_ARGS: dict[str, str] = _config["required_args"]
+
 # Extract default arguments
 DEFAULT_ARGS: dict[str, str] = _config["default_args"]

vec_inf/client/_utils.py

@@ -14,9 +14,9 @@ from typing import Any, Optional, Union, cast
 import requests
 import yaml
 
-from vec_inf.client._client_vars import MODEL_READY_SIGNATURE, REQUIRED_ARGS
+from vec_inf.client._client_vars import MODEL_READY_SIGNATURE
 from vec_inf.client._exceptions import MissingRequiredFieldsError
-from vec_inf.client._slurm_vars import CACHED_CONFIG_DIR
+from vec_inf.client._slurm_vars import CACHED_CONFIG_DIR, REQUIRED_ARGS
 from vec_inf.client.config import ModelConfig
 from vec_inf.client.models import ModelStatus
 

vec_inf/config/environment.yaml

@@ -15,6 +15,10 @@ allowed_values:
   partition: []
   resource_type: ["l40s", "h100"]
 
+required_args:
+  account: "VEC_INF_ACCOUNT"
+  work_dir: "VEC_INF_WORK_DIR"
+
 default_args:
   cpus_per_task: "16"
   mem_per_node: "64G"

{vec_inf-0.7.0.dist-info → vec_inf-0.7.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vec-inf
-Version: 0.7.0
+Version: 0.7.1
 Summary: Efficient LLM inference on Slurm clusters using vLLM.
 Author-email: Marshall Wang <marshall.wang@vectorinstitute.ai>
 License-Expression: MIT
@@ -66,7 +66,7 @@ You should see an output like the following:
 
 <img width="720" alt="launch_image" src="https://github.com/user-attachments/assets/c1e0c60c-cf7a-49ed-a426-fdb38ebf88ee" />
 
-**NOTE**: On Vector Killarney Cluster environment, the following fields are required:
+**NOTE**: You can set the required fields in the environment configuration (`environment.yaml`), it's a mapping between required arguments and their corresponding environment variables. On the Vector **Killarney** Cluster environment, the required fields are:
   * `--account`, `-A`: The Slurm account, this argument can be set to default by setting environment variable `VEC_INF_ACCOUNT`.
   * `--work-dir`, `-D`: A working directory other than your home directory, this argument can be set to default by seeting environment variable `VEC_INF_WORK_DIR`.
 
@@ -96,6 +96,11 @@ Example:
 >>> status = client.get_status(job_id)
 >>> if status.status == ModelStatus.READY:
 ...     print(f"Model is ready at {status.base_url}")
+>>> # Alternatively, use wait_until_ready which will either return a StatusResponse or throw a ServerError
+>>> try:
+>>>     status = wait_until_ready(job_id)
+>>> except ServerError as e:
+>>>     print(f"Model launch failed: {e}")
 >>> client.shutdown_model(job_id)
 ```
 
@@ -146,6 +151,19 @@ Once the inference server is ready, you can start sending in inference requests.
 ## SSH tunnel from your local device
 If you want to run inference from your local device, you can open a SSH tunnel to your cluster environment like the following:
 ```bash
-ssh -L 8081:172.17.8.29:8081 username@v.vectorinstitute.ai -N
+ssh -L 8081:10.1.1.29:8081 username@v.vectorinstitute.ai -N
+```
+The example provided above is for the Vector Killarney cluster, change the variables accordingly for your environment. The IP address for the compute nodes on Killarney follow `10.1.1.XX` pattern, where `XX` is the GPU number (`kn029` -> `29` in this example).
+
+## Reference
+If you found Vector Inference useful in your research or applications, please cite using the following BibTeX template:
+```
+@software{vector_inference,
+  title        = {Vector Inference: Efficient LLM inference on Slurm clusters using vLLM},
+  author       = {Wang, Marshall},
+  organization = {Vector Institute},
+  year         = {<YEAR_OF_RELEASE>},
+  version      = {<VERSION_TAG>},
+  url          = {https://github.com/VectorInstitute/vector-inference}
+}
 ```
-Where the last number in the URL is the GPU number (gpu029 in this case). The example provided above is for the vector cluster, change the variables accordingly for your environment

{vec_inf-0.7.0.dist-info → vec_inf-0.7.1.dist-info}/RECORD

@@ -7,21 +7,21 @@ vec_inf/cli/_helper.py,sha256=Jr9NnMhGflkx3YEfYCN1rMHQgUzMAAwlSx_BLH92tVM,16511
 vec_inf/cli/_utils.py,sha256=23vSbmvNOWY1-W1aOAwYqNDkDDmx-5UVlCiXAtxUZ8A,1057
 vec_inf/cli/_vars.py,sha256=V6DrJs_BuUa4yNcbBSSnMwpcyXwEBsizy3D0ubIg2fA,777
 vec_inf/client/__init__.py,sha256=OLlUJ4kL1R-Kh-nXNbvKlAZ3mtHcnozHprVufkVCNWk,739
-vec_inf/client/_client_vars.py,sha256=qt47xQyZX2YcBtxk5qqmsE6qM5c3m8E2RhRBa2AY068,2619
+vec_inf/client/_client_vars.py,sha256=1D-bX9dS0-pFImLvgWt2hUnwJiz-VaxuLb2HIfPML8I,2408
 vec_inf/client/_exceptions.py,sha256=94Nx_5k1SriJNXzbdnwyXFZolyMutydU08Gsikawzzo,749
 vec_inf/client/_helper.py,sha256=P8A9JHRMzxJRl0dgTuv9xfOluEV3BthUM1KzQlWkR7E,35752
 vec_inf/client/_slurm_script_generator.py,sha256=d2NowdKMQR1lsVI_hw9ObKC3uSk8YJr75ZYRMkvp0RA,13354
 vec_inf/client/_slurm_templates.py,sha256=TAH-wQV4gP2CCwxP3BmShebohtSmlMstlJT9QK6n4Dc,8277
-vec_inf/client/_slurm_vars.py,sha256=9BGA4Y4dGzXez6FG4V53GsMlHb9xOj7W1d7ANjkTvSQ,2723
-vec_inf/client/_utils.py,sha256=aQoPFYUNjp0OGHDdvPu1oec_Eslv0PjtKAiW54WSgAo,12593
+vec_inf/client/_slurm_vars.py,sha256=sgP__XhpE1K7pvOzVFmotUXmINYPcOuFP-zGaePT5Iw,2910
+vec_inf/client/_utils.py,sha256=XamAz8-AJELgkXHrR082ptTsbHSiWI47SY6MlXA44rU,12593
 vec_inf/client/api.py,sha256=pkgNE37r7LzYBDjRGAKAh7rhOUMKHGwghJh6Hfb45TI,11681
 vec_inf/client/config.py,sha256=VU4h2iqL0rxYAqGw2HBF_l6QvvSDJy5M79IgX5G2PW4,5830
 vec_inf/client/models.py,sha256=qxLxsVoEhxNkuCmtABqs8In5erkwTZDK0wih7U2_U38,7296
 vec_inf/config/README.md,sha256=TvZOqZyTUaAFr71hC7GVgg6QUw80AXREyq8wS4D-F30,528
-vec_inf/config/environment.yaml,sha256=VBBlHx6zbYnzjwhWcsUI6m5Xqc-2KLPOr1oZ6GUlIWk,602
+vec_inf/config/environment.yaml,sha256=oEDp85hUERJO9NNn4wYhcgunnmkln50GNHDzG_3isMw,678
 vec_inf/config/models.yaml,sha256=vzAOqEu6M_lXput83MAhNzj-aNGSBzjbC6LydOmNqxk,26248
-vec_inf-0.7.0.dist-info/METADATA,sha256=4JtnZxIZA1QXN6m5YsMEUWxb_HjKGgnNBFGf8Pe-IuI,9088
-vec_inf-0.7.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-vec_inf-0.7.0.dist-info/entry_points.txt,sha256=uNRXjCuJSR2nveEqD3IeMznI9oVI9YLZh5a24cZg6B0,49
-vec_inf-0.7.0.dist-info/licenses/LICENSE,sha256=mq8zeqpvVSF1EsxmydeXcokt8XnEIfSofYn66S2-cJI,1073
-vec_inf-0.7.0.dist-info/RECORD,,
+vec_inf-0.7.1.dist-info/METADATA,sha256=CJEnzc3VLXxJ_00I1ubtwNNZQjvafddxlJyoi_bSwpo,10047
+vec_inf-0.7.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+vec_inf-0.7.1.dist-info/entry_points.txt,sha256=uNRXjCuJSR2nveEqD3IeMznI9oVI9YLZh5a24cZg6B0,49
+vec_inf-0.7.1.dist-info/licenses/LICENSE,sha256=mq8zeqpvVSF1EsxmydeXcokt8XnEIfSofYn66S2-cJI,1073
+vec_inf-0.7.1.dist-info/RECORD,,