decisionnode-edge 0.9.2__tar.gz → 0.9.2.1__tar.gz

{decisionnode_edge-0.9.2 → decisionnode_edge-0.9.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: decisionnode-edge
-Version: 0.9.2
+Version: 0.9.2.1
 Summary: Run DecisionNode Edge locally: typed, calibrated decisions in the shape of the DecisionNode API.
 Author-email: "Bynn Intelligence Inc." <licensing@bynn.com>
 License-Expression: AGPL-3.0-only
@@ -230,6 +230,7 @@ Options can be flat (`{"threads": 16}`) or under a backend's name (`{"openvino":
 | `llamacpp` | `server`, `gguf`: the program and the model file | `llama-server` on `PATH`, the build's file |
 | `llamacpp` | `gpu_layers`, `threads` | 999 (all layers on the GPU when there is one), llama.cpp's choice |
 | `llamacpp` | `context`: the longest prompt llama.cpp takes | 16,384 |
+| `mlx` | `cache_limit_gb`: how much freed GPU memory MLX keeps for reuse | 2 |
 
 vLLM runs its engine inside your process and uses the first visible GPU (choose it with `CUDA_VISIBLE_DEVICES`). Its first load takes about half a minute while it compiles. MLX uses the Apple GPU.
 

{decisionnode_edge-0.9.2 → decisionnode_edge-0.9.2.1}/README.md

@@ -204,6 +204,7 @@ Options can be flat (`{"threads": 16}`) or under a backend's name (`{"openvino":
 | `llamacpp` | `server`, `gguf`: the program and the model file | `llama-server` on `PATH`, the build's file |
 | `llamacpp` | `gpu_layers`, `threads` | 999 (all layers on the GPU when there is one), llama.cpp's choice |
 | `llamacpp` | `context`: the longest prompt llama.cpp takes | 16,384 |
+| `mlx` | `cache_limit_gb`: how much freed GPU memory MLX keeps for reuse | 2 |
 
 vLLM runs its engine inside your process and uses the first visible GPU (choose it with `CUDA_VISIBLE_DEVICES`). Its first load takes about half a minute while it compiles. MLX uses the Apple GPU.
 

{decisionnode_edge-0.9.2 → decisionnode_edge-0.9.2.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "decisionnode-edge"
-version = "0.9.2"
+version = "0.9.2.1"
 description = "Run DecisionNode Edge locally: typed, calibrated decisions in the shape of the DecisionNode API."
 readme = "README.md"
 requires-python = ">=3.10"

{decisionnode_edge-0.9.2 → decisionnode_edge-0.9.2.1}/src/decisionnode_edge/__init__.py

@@ -10,4 +10,4 @@ from .model import EdgeModel
 from .request import RequestError
 
 __all__ = ["EdgeModel", "RequestError"]
-__version__ = "0.9.2"
+__version__ = "0.9.2.1"

{decisionnode_edge-0.9.2 → decisionnode_edge-0.9.2.1}/src/decisionnode_edge/backends.py

@@ -165,7 +165,7 @@ def load(name: str, path: str, config: dict, artifacts: str | None, options: dic
     if name == "vllm":
         return VLLM(path, max_tokens, options, device)
     if name == "mlx":
-        return MLX(path)
+        return MLX(path, options)
     build = read_build(artifacts)
     if build is not None and build.get("model_fingerprint") != fingerprint(path):
         raise ValueError(f"{artifacts} was built from another model than {path}")
@@ -304,14 +304,18 @@ class MLX:
     touches readout.safetensors); the text model's output is the hidden state after the final norm."""
 
     name = "mlx"
+    cache_limit_gb = 2.0
 
-    def __init__(self, path: str):
+    def __init__(self, path: str, options: dict | None = None):
         from pathlib import Path
 
         import mlx.core as mx
         from mlx_lm.utils import load_model
 
         self.mx = mx
+        # MLX keeps freed buffers for reuse, by default up to its memory limit, so a server reading long prompts grew its
+        # cache to most of the machine's memory. A bound keeps the reuse that matters for speed.
+        mx.set_cache_limit(int(float((options or {}).get("cache_limit_gb", self.cache_limit_gb)) * 2**30))
         model, _ = load_model(Path(path))
         self.text = model.language_model.model
 

{decisionnode_edge-0.9.2 → decisionnode_edge-0.9.2.1}/src/decisionnode_edge.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: decisionnode-edge
-Version: 0.9.2
+Version: 0.9.2.1
 Summary: Run DecisionNode Edge locally: typed, calibrated decisions in the shape of the DecisionNode API.
 Author-email: "Bynn Intelligence Inc." <licensing@bynn.com>
 License-Expression: AGPL-3.0-only
@@ -230,6 +230,7 @@ Options can be flat (`{"threads": 16}`) or under a backend's name (`{"openvino":
 | `llamacpp` | `server`, `gguf`: the program and the model file | `llama-server` on `PATH`, the build's file |
 | `llamacpp` | `gpu_layers`, `threads` | 999 (all layers on the GPU when there is one), llama.cpp's choice |
 | `llamacpp` | `context`: the longest prompt llama.cpp takes | 16,384 |
+| `mlx` | `cache_limit_gb`: how much freed GPU memory MLX keeps for reuse | 2 |
 
 vLLM runs its engine inside your process and uses the first visible GPU (choose it with `CUDA_VISIBLE_DEVICES`). Its first load takes about half a minute while it compiles. MLX uses the Apple GPU.