sendnn-inference 2.2.2__tar.gz → 2.2.4__tar.gz

{sendnn_inference-2.2.2 → sendnn_inference-2.2.4}/.github/ci_model_cache.yaml

@@ -13,5 +13,3 @@ models:
     revision: cf74d8acd4f198de950bf004b262e6accfed5d2c
   - repo: cross-encoder/stsb-roberta-large
     revision: 2b12c2c0088918e76151fd5937b7bba986ef1f98
-  - repo: Qwen/Qwen3-Embedding-0.6B
-    revision: 97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3

{sendnn_inference-2.2.2 → sendnn_inference-2.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sendnn-inference
-Version: 2.2.2
+Version: 2.2.4
 Summary: vLLM plugin for Spyre hardware support
 License: Apache 2
 Requires-Python: >=3.11

{sendnn_inference-2.2.2 → sendnn_inference-2.2.4}/docs/mkdocs/hooks/generate_model_tables.py

@@ -21,6 +21,8 @@ EXCLUDED_MODELS = {
     "ibm-granite/granite-4-8b-dense-hybrid",
     "ibm-granite/granite-4-8b-dense",
     "ibm-granite/granite-4-8b-dense-FP8",
+    "Qwen/Qwen3-Embedding-0.6B",
+    "Qwen/Qwen3-Embedding-4B",
 }
 
 

{sendnn_inference-2.2.2 → sendnn_inference-2.2.4}/sendnn_inference/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '2.2.2'
-__version_tuple__ = version_tuple = (2, 2, 2)
+__version__ = version = '2.2.4'
+__version_tuple__ = version_tuple = (2, 2, 4)
 
-__commit_id__ = commit_id = 'gd054d78'
+__commit_id__ = commit_id = 'gc694b58'

{sendnn_inference-2.2.2 → sendnn_inference-2.2.4}/sendnn_inference/config/model_configs.yaml

@@ -338,30 +338,6 @@ models:
           - prompt_len: 512
             batch_size: 64
 
-  Qwen/Qwen3-Embedding-0.6B:
-    architecture:
-      model_type: qwen3
-      num_hidden_layers: 28
-      vocab_size: 151669
-
-    static_batching_configs:
-      - tp_size: 1
-        warmup_shapes:
-          - prompt_len: 512
-            batch_size: 64
-
-  Qwen/Qwen3-Embedding-4B:
-    architecture:
-      model_type: qwen3
-      num_hidden_layers: 36
-      vocab_size: 151665
-
-    static_batching_configs:
-      - tp_size: 1
-        warmup_shapes:
-          - prompt_len: 512
-            batch_size: 64
-
   # Other supported models (static batching only)
   intfloat/multilingual-e5-large:
     architecture:

{sendnn_inference-2.2.2 → sendnn_inference-2.2.4}/sendnn_inference/envs.py

@@ -52,12 +52,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
     # for, formatted as comma separated list. Only applicable in pooling.
     "SENDNN_INFERENCE_WARMUP_PROMPT_LENS": lambda: [
         int(p)
-        for p in os.getenv(key="SENDNN_INFERENCE_WARMUP_PROMPT_LENS", default="64").split(",")
+        for p in os.getenv(key="SENDNN_INFERENCE_WARMUP_PROMPT_LENS", default="512").split(",")
     ],
     # Defines the batch sizes the Spyre accelerator should be prepared
     # for, formatted as comma separated list. Only applicable in pooling.
     "SENDNN_INFERENCE_WARMUP_BATCH_SIZES": lambda: [
-        int(b) for b in os.getenv(key="SENDNN_INFERENCE_WARMUP_BATCH_SIZES", default="1").split(",")
+        int(b) for b in os.getenv(key="SENDNN_INFERENCE_WARMUP_BATCH_SIZES", default="8").split(",")
     ],
     # Defines the backend that torch.compile will use when using Spyre
     # Available options:

{sendnn_inference-2.2.2 → sendnn_inference-2.2.4}/sendnn_inference/model_executor/model_loader/spyre.py

@@ -117,7 +117,7 @@ class SpyreCausalLM(nn.Module):
             self.parallel_config
         )
 
-        if self.config.model_type in {"llama", "granite", "granitemoehybrid"}:
+        if self.config.model_type in {"llama", "granite", "granitemoehybrid", "qwen3"}:
             self.kv_cache_specs["num_layers"] = self.config.num_hidden_layers
             self.kv_cache_specs["head_dim"] = getattr(
                 self.fms_model.config,

{sendnn_inference-2.2.2 → sendnn_inference-2.2.4}/sendnn_inference/platform.py

@@ -1,4 +1,5 @@
 import sys
+import platform
 from string import Template
 import multiprocessing
 import importlib.metadata
@@ -56,6 +57,10 @@ THREADING_ENVS = [
     "MKL_NUM_THREADS",
 ]
 
+DEFAULT_MAX_MODEL_LEN = 32 * 1024
+DEFAULT_MAX_NUM_SEQS = 32
+DEFAULT_TKV_LIMIT = 131072  # 128k
+
 
 # Needed by vllm/model_executor/layers/pooler.py:562
 # Copied from vllm/utils/__init__.py
@@ -327,7 +332,21 @@ class SpyrePlatform(Platform):
                         f"{error_msg}. SENDNN_INFERENCE_REQUIRE_KNOWN_CONFIG is set, "
                         "which requires a known configuration to be found."
                     )
-                logger.debug(error_msg)
+                logger.info(
+                    "%s. Capping max-num-seqs at %d and max-model-len at %d",
+                    error_msg,
+                    DEFAULT_MAX_NUM_SEQS,
+                    DEFAULT_MAX_MODEL_LEN,
+                )
+                if is_decoder:
+                    vllm_config.scheduler_config.max_num_seqs = min(
+                        vllm_config.scheduler_config.max_num_seqs, DEFAULT_MAX_NUM_SEQS
+                    )
+                    vllm_config.model_config.max_model_len = min(
+                        vllm_config.model_config.max_model_len, DEFAULT_MAX_MODEL_LEN
+                    )
+                    tkv_env = int(os.getenv("VLLM_DT_MAX_BATCH_TKV_LIMIT", f"{DEFAULT_TKV_LIMIT}"))
+                    os.environ["VLLM_DT_MAX_BATCH_TKV_LIMIT"] = str(min(DEFAULT_TKV_LIMIT, tkv_env))
 
         else:
             logger.debug(
@@ -643,12 +662,16 @@ class SpyrePlatform(Platform):
 
         # NOTE: math.ceil can output a number for each worker that sums
         # to a total greater than cpu_count.
-        thread_factor = worker_count
         if cls._config.model_config.is_multimodal_model:
-            # thread_factor value/formula subject to further tuning
-            thread_factor = 1
-
-        cpus_per_worker = math.ceil(cpu_count / thread_factor) if cpu_count is not None else None
+            if platform.machine() == "ppc64le":
+                cpus_per_worker = (
+                    min(psutil.cpu_count(logical=True), 36) if cpu_count is not None else None
+                )
+            else:
+                # Formula for cpus_per_worker can be adjusted per architecture
+                cpus_per_worker = math.ceil(cpu_count) if cpu_count is not None else None
+        else:
+            cpus_per_worker = math.ceil(cpu_count / worker_count) if cpu_count is not None else None
 
         thread_warning = (
             "Excessive threads may result in CPU contention. "

{sendnn_inference-2.2.2 → sendnn_inference-2.2.4}/sendnn_inference/v1/worker/spyre_model_runner.py

@@ -324,7 +324,9 @@ class SpyrePoolingModelRunner(
                 )
                 self._model = self._model.base_model
             else:
-                self._model = AutoModel.from_pretrained(self.model_config.model)
+                self._model = AutoModel.from_pretrained(
+                    self.model_config.model, dtype=torch.float16
+                )
         elif task == "classify":
             class_model = AutoModelForSequenceClassification.from_pretrained(
                 self.model_config.model

{sendnn_inference-2.2.2 → sendnn_inference-2.2.4}/sendnn_inference.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sendnn-inference
-Version: 2.2.2
+Version: 2.2.4
 Summary: vLLM plugin for Spyre hardware support
 License: Apache 2
 Requires-Python: >=3.11

{sendnn_inference-2.2.2 → sendnn_inference-2.2.4}/tests/spyre_util.py

@@ -275,10 +275,6 @@ register_model_info(
     name="sentence-transformers/all-roberta-large-v1",
     revision="cf74d8acd4f198de950bf004b262e6accfed5d2c",
 )
-register_model_info(
-    name="Qwen/Qwen3-Embedding-0.6B",
-    revision="97b0c614be4d77ee51c0cef4e5f07c00f9eb65b3",
-)
 register_model_info(
     name="cross-encoder/stsb-roberta-large",
     revision="2b12c2c0088918e76151fd5937b7bba986ef1f98",
@@ -324,10 +320,8 @@ def _default_test_models(
     """Return the default set of test models as pytest parameterizations"""
     if isEmbeddings:
         roberta = REFERENCE_MODELS["sentence-transformers/all-roberta-large-v1"]
-        qwen3 = REFERENCE_MODELS["Qwen/Qwen3-Embedding-0.6B"]
         params = [
             pytest.param(roberta, marks=[pytest.mark.embedding], id=roberta.name),
-            pytest.param(qwen3, marks=[pytest.mark.embedding], id=qwen3.name),
         ]
         return params