pyinferencemanager 1.2.0__tar.gz → 1.3.0__tar.gz

{pyinferencemanager-1.2.0 → pyinferencemanager-1.3.0}/Cargo.lock

@@ -1339,7 +1339,7 @@ checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
 
 [[package]]
 name = "pyinferencemanager"
-version = "1.2.0"
+version = "1.3.0"
 dependencies = [
  "chrono",
  "pyinferencemanager-core",
@@ -1350,7 +1350,7 @@ dependencies = [
 
 [[package]]
 name = "pyinferencemanager-core"
-version = "1.2.0"
+version = "1.3.0"
 dependencies = [
  "anyhow",
  "async-trait",

{pyinferencemanager-1.2.0 → pyinferencemanager-1.3.0}/Cargo.toml

@@ -3,7 +3,7 @@ members = ["crates/pyinferencemanager-core", "crates/pyinferencemanager-py"]
 resolver = "2"
 
 [workspace.package]
-version = "1.2.0"
+version = "1.3.0"
 edition = "2021"
 rust-version = "1.82"
 authors = ["Georgi Mammen Mullassery <mullassery@gmail.com>"]

{pyinferencemanager-1.2.0 → pyinferencemanager-1.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyinferencemanager
-Version: 1.2.0
+Version: 1.3.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Environment :: Console
 Classifier: Intended Audience :: Developers
@@ -134,8 +134,9 @@ Real HTTP calls, with retry/backoff and cost tracking:
 | OpenAI | `OPENAI_API_KEY` | `gpt-4o-mini` |
 | Google Gemini | `GEMINI_API_KEY` / `GOOGLE_API_KEY` | `gemini-1.5-flash` |
 | Ollama (local) | — | whatever models you've pulled locally |
+| vLLM (local) | — | OpenAI-compatible endpoint, default `localhost:8000` |
 
-Additionally, `vllm`, `tensorrt_llm`, `mlc_llm`, and `colibri` exist as **cost-estimator-only stubs** (`BackendKind`/`RuntimeBackend` trait implementations with real cost/latency estimation logic, but no live inference) — they're placeholders for self-hosted inference servers, not currently wired to make real calls. `orchestrator.available_backends()` lists all of these honestly, including the stubs.
+Additionally, `tensorrt_llm`, `mlc_llm`, and `colibri` exist as **cost-estimator-only stubs** (`BackendKind`/`RuntimeBackend` trait implementations with real cost/latency estimation logic, but no live inference) — they're placeholders for self-hosted inference servers, not currently wired to make real calls. `orchestrator.available_backends()` lists all of these honestly, including the stubs.
 
 Cloud execution dispatches through `BackendRegistry`/`RuntimeBackend` (not a hand-matched provider enum): `ProviderExecutor::execute` maps a `CloudProvider` to its `BackendKind`, registers the one backend it needs (reading the API key from the env vars above), and calls it through the same `RuntimeBackend::infer` trait object every backend implements — so adding a new cloud provider is a new `BackendKind`/backend + one dispatch arm, not edits scattered across every call site that used to match the old enum.
 
@@ -152,7 +153,7 @@ The network connector (`_mcp_connector.InferenceManager.start_mcp_connector()`)
 
 ## Known issues
 
-- `vllm`, `tensorrt_llm`, `mlc_llm`, and `colibri` backends are cost-estimator-only stubs (see [Providers](#providers)) — they do not make live inference calls yet.
+- `tensorrt_llm`, `mlc_llm`, and `colibri` backends are cost-estimator-only stubs (see [Providers](#providers)) — they do not make live inference calls yet. `vllm` was a stub too as of 1.2.0 but is now a real backend.
 - No open GitHub issues and no `TODO`/`FIXME` markers in the codebase at the time of this writing.
 
 ## License

{pyinferencemanager-1.2.0 → pyinferencemanager-1.3.0}/README.md

@@ -100,8 +100,9 @@ Real HTTP calls, with retry/backoff and cost tracking:
 | OpenAI | `OPENAI_API_KEY` | `gpt-4o-mini` |
 | Google Gemini | `GEMINI_API_KEY` / `GOOGLE_API_KEY` | `gemini-1.5-flash` |
 | Ollama (local) | — | whatever models you've pulled locally |
+| vLLM (local) | — | OpenAI-compatible endpoint, default `localhost:8000` |
 
-Additionally, `vllm`, `tensorrt_llm`, `mlc_llm`, and `colibri` exist as **cost-estimator-only stubs** (`BackendKind`/`RuntimeBackend` trait implementations with real cost/latency estimation logic, but no live inference) — they're placeholders for self-hosted inference servers, not currently wired to make real calls. `orchestrator.available_backends()` lists all of these honestly, including the stubs.
+Additionally, `tensorrt_llm`, `mlc_llm`, and `colibri` exist as **cost-estimator-only stubs** (`BackendKind`/`RuntimeBackend` trait implementations with real cost/latency estimation logic, but no live inference) — they're placeholders for self-hosted inference servers, not currently wired to make real calls. `orchestrator.available_backends()` lists all of these honestly, including the stubs.
 
 Cloud execution dispatches through `BackendRegistry`/`RuntimeBackend` (not a hand-matched provider enum): `ProviderExecutor::execute` maps a `CloudProvider` to its `BackendKind`, registers the one backend it needs (reading the API key from the env vars above), and calls it through the same `RuntimeBackend::infer` trait object every backend implements — so adding a new cloud provider is a new `BackendKind`/backend + one dispatch arm, not edits scattered across every call site that used to match the old enum.
 
@@ -118,7 +119,7 @@ The network connector (`_mcp_connector.InferenceManager.start_mcp_connector()`)
 
 ## Known issues
 
-- `vllm`, `tensorrt_llm`, `mlc_llm`, and `colibri` backends are cost-estimator-only stubs (see [Providers](#providers)) — they do not make live inference calls yet.
+- `tensorrt_llm`, `mlc_llm`, and `colibri` backends are cost-estimator-only stubs (see [Providers](#providers)) — they do not make live inference calls yet. `vllm` was a stub too as of 1.2.0 but is now a real backend.
 - No open GitHub issues and no `TODO`/`FIXME` markers in the codebase at the time of this writing.
 
 ## License

{pyinferencemanager-1.2.0 → pyinferencemanager-1.3.0}/crates/pyinferencemanager-core/src/backends/mod.rs

@@ -7,6 +7,7 @@ pub mod gemini_backend;
 pub mod ollama_backend;
 pub mod openai_backend;
 pub mod stub_backend;
+pub mod vllm_backend;
 
 pub use anthropic_backend::AnthropicBackend;
 pub use colibri_backend::ColibriBackend;
@@ -14,6 +15,7 @@ pub use gemini_backend::GeminiBackend;
 pub use ollama_backend::OllamaBackend;
 pub use openai_backend::OpenAiBackend;
 pub use stub_backend::StubBackend;
+pub use vllm_backend::VLlmBackend;
 
 #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)]
 pub enum BackendKind {
@@ -105,7 +107,7 @@ pub enum AnyBackend {
     OpenAi(OpenAiBackend),
     Gemini(GeminiBackend),
     Ollama(OllamaBackend),
-    VLlm(StubBackend),
+    VLlm(VLlmBackend),
     TensorRtLlm(StubBackend),
     MlcLlm(StubBackend),
     Colibri(ColibriBackend),

{pyinferencemanager-1.2.0 → pyinferencemanager-1.3.0}/crates/pyinferencemanager-core/src/backends/stub_backend.rs

@@ -1,5 +1,10 @@
 use super::{BackendKind, InferenceRequest, InferenceResult, ModelProfile, RuntimeBackend};
 
+/// Placeholder `RuntimeBackend` for runtimes that don't have a real client
+/// implementation yet. `BackendKind::VLlm` used to be one of these -- it now
+/// has a real implementation in `vllm_backend::VLlmBackend` (see
+/// `AnyBackend::VLlm` in `mod.rs`), so it's no longer constructed with this
+/// stub. Only `TensorRtLlm` and `MlcLlm` remain genuinely unimplemented.
 pub struct StubBackend {
     kind: BackendKind,
 }
@@ -22,15 +27,6 @@ impl RuntimeBackend for StubBackend {
     fn estimate_cost(&self, profile: &ModelProfile) -> f32 {
         let size_gb = profile.size_gb();
         match self.kind {
-            BackendKind::VLlm => {
-                if size_gb < 20.0 {
-                    0.0
-                } else if size_gb < 50.0 {
-                    0.00001
-                } else {
-                    0.00002
-                }
-            }
             BackendKind::TensorRtLlm => {
                 if size_gb < 20.0 {
                     0.0
@@ -52,10 +48,6 @@ impl RuntimeBackend for StubBackend {
     fn estimate_latency(&self, profile: &ModelProfile) -> u64 {
         let size_gb = profile.size_gb();
         match self.kind {
-            BackendKind::VLlm => {
-                let base = (size_gb * 30.0) as u64;
-                base.max(200).min(30000)
-            }
             BackendKind::TensorRtLlm => {
                 let base = (size_gb * 20.0) as u64;
                 base.max(150).min(20000)
@@ -79,7 +71,7 @@ mod tests {
 
     #[test]
     fn test_stub_backend_infer_returns_error() {
-        let backend = StubBackend::new(BackendKind::VLlm);
+        let backend = StubBackend::new(BackendKind::TensorRtLlm);
         let request = InferenceRequest {
             model: "model".to_string(),
             prompt: "test".to_string(),
@@ -93,23 +85,21 @@ mod tests {
 
     #[test]
     fn test_stub_backend_kind() {
-        let vllm = StubBackend::new(BackendKind::VLlm);
         let trt = StubBackend::new(BackendKind::TensorRtLlm);
         let mlc = StubBackend::new(BackendKind::MlcLlm);
 
-        assert_eq!(vllm.kind(), BackendKind::VLlm);
         assert_eq!(trt.kind(), BackendKind::TensorRtLlm);
         assert_eq!(mlc.kind(), BackendKind::MlcLlm);
     }
 
     #[test]
     fn test_stub_backend_estimate_cost() {
-        let vllm = StubBackend::new(BackendKind::VLlm);
+        let trt = StubBackend::new(BackendKind::TensorRtLlm);
         let small = ModelProfile::new(10 * 1_073_741_824, "model".to_string());
         let large = ModelProfile::new(60 * 1_073_741_824, "model".to_string());
 
-        let small_cost = vllm.estimate_cost(&small);
-        let large_cost = vllm.estimate_cost(&large);
+        let small_cost = trt.estimate_cost(&small);
+        let large_cost = trt.estimate_cost(&large);
 
         assert!(small_cost <= large_cost);
     }

pyinferencemanager-1.3.0/crates/pyinferencemanager-core/src/backends/vllm_backend.rs

@@ -0,0 +1,155 @@
+use super::{BackendKind, InferenceRequest, InferenceResult, ModelProfile, RuntimeBackend};
+use crate::engines::VLlmClient;
+
+/// Real `RuntimeBackend` for a self-hosted vLLM server, talking to its
+/// OpenAI-compatible HTTP API (`POST /v1/completions`). Follows the same
+/// shape as `OllamaBackend`: a thin adapter that owns just enough config
+/// (here, the base URL and optional API key) to construct a fresh client
+/// per call and translate `InferenceRequest`/`InferenceResult` to and from
+/// the wire format.
+pub struct VLlmBackend {
+    base_url: String,
+    api_key: Option<String>,
+}
+
+impl VLlmBackend {
+    pub fn new(base_url: String) -> Self {
+        VLlmBackend {
+            base_url,
+            api_key: None,
+        }
+    }
+
+    pub fn with_api_key(mut self, api_key: Option<String>) -> Self {
+        self.api_key = api_key;
+        self
+    }
+}
+
+#[async_trait::async_trait]
+impl RuntimeBackend for VLlmBackend {
+    async fn infer(&self, request: InferenceRequest) -> crate::Result<InferenceResult> {
+        let start = std::time::Instant::now();
+        let client = VLlmClient::new(&self.base_url).with_api_key(self.api_key.clone());
+
+        let response = client
+            .complete(&request.model, &request.prompt, request.max_tokens)
+            .await?;
+
+        let latency_ms = start.elapsed().as_millis() as u64;
+
+        Ok(InferenceResult {
+            output: response.text,
+            tokens_used: response.tokens_used,
+            latency_ms,
+        })
+    }
+
+    // vLLM is a self-hosted runtime like Ollama, not a metered cloud API --
+    // there's no per-token bill from vLLM itself. What scales with model
+    // size is the GPU/compute cost of running it, so this mirrors the
+    // original stub's tiered estimate (kept identical to avoid silently
+    // changing cost-comparison behavior elsewhere, e.g. CostEstimator)
+    // rather than collapsing to a flat 0.0 like OllamaBackend, since vLLM is
+    // typically deployed for larger models where that compute cost is
+    // material.
+    fn estimate_cost(&self, profile: &ModelProfile) -> f32 {
+        let size_gb = profile.size_gb();
+        if size_gb < 20.0 {
+            0.0
+        } else if size_gb < 50.0 {
+            0.00001
+        } else {
+            0.00002
+        }
+    }
+
+    fn estimate_latency(&self, profile: &ModelProfile) -> u64 {
+        let size_gb = profile.size_gb();
+        let base = (size_gb * 30.0) as u64;
+        base.max(200).min(30000)
+    }
+
+    fn kind(&self) -> BackendKind {
+        BackendKind::VLlm
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_vllm_backend_kind() {
+        let backend = VLlmBackend::new("http://localhost:8000".to_string());
+        assert_eq!(backend.kind(), BackendKind::VLlm);
+    }
+
+    #[test]
+    fn test_vllm_estimate_cost_tiers_by_model_size() {
+        let backend = VLlmBackend::new("http://localhost:8000".to_string());
+        let small = ModelProfile::new(10 * 1_073_741_824, "model".to_string());
+        let medium = ModelProfile::new(30 * 1_073_741_824, "model".to_string());
+        let large = ModelProfile::new(60 * 1_073_741_824, "model".to_string());
+
+        assert_eq!(backend.estimate_cost(&small), 0.0);
+        assert!(backend.estimate_cost(&medium) < backend.estimate_cost(&large));
+    }
+
+    #[test]
+    fn test_vllm_estimate_latency_bounds() {
+        let backend = VLlmBackend::new("http://localhost:8000".to_string());
+        let small_profile = ModelProfile::new(7 * 1_073_741_824, "llama".to_string());
+        let large_profile = ModelProfile::new(70 * 1_073_741_824, "llama".to_string());
+
+        let small_latency = backend.estimate_latency(&small_profile);
+        let large_latency = backend.estimate_latency(&large_profile);
+
+        assert!(small_latency < large_latency);
+        assert!(small_latency >= 200);
+    }
+
+    use wiremock::matchers::{method, path};
+    use wiremock::{Mock, MockServer, ResponseTemplate};
+
+    #[tokio::test]
+    async fn test_infer_real_http_dispatch_through_backend() {
+        let mock_server = MockServer::start().await;
+
+        Mock::given(method("POST"))
+            .and(path("/v1/completions"))
+            .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
+                "choices": [{"text": "Hello from vLLM", "finish_reason": "stop"}],
+                "usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7}
+            })))
+            .mount(&mock_server)
+            .await;
+
+        let backend = VLlmBackend::new(mock_server.uri());
+        let request = InferenceRequest {
+            model: "meta-llama/Llama-3-8b".to_string(),
+            prompt: "Hi".to_string(),
+            max_tokens: 50,
+        };
+
+        let result = backend.infer(request).await.expect("mocked infer should succeed");
+        assert_eq!(result.output, "Hello from vLLM");
+        assert_eq!(result.tokens_used, 4);
+    }
+
+    #[tokio::test]
+    async fn test_infer_propagates_backend_unreachable_error() {
+        // Nothing listening -- must return a real error, not silently
+        // succeed or hang, since this is meant to replace the old stub that
+        // always errored with "not configured".
+        let backend = VLlmBackend::new("http://127.0.0.1:1".to_string());
+        let request = InferenceRequest {
+            model: "model".to_string(),
+            prompt: "Hi".to_string(),
+            max_tokens: 10,
+        };
+
+        let result = backend.infer(request).await;
+        assert!(result.is_err());
+    }
+}

{pyinferencemanager-1.2.0 → pyinferencemanager-1.3.0}/crates/pyinferencemanager-core/src/engines/mod.rs

@@ -3,9 +3,11 @@ pub mod gemini_client;
 pub mod ollama_client;
 pub mod openai_client;
 pub mod provider_health;
+pub mod vllm_client;
 
 pub use cloud_client::CloudClient;
 pub use gemini_client::GeminiClient;
 pub use ollama_client::OllamaClient;
 pub use openai_client::OpenAIClient;
 pub use provider_health::{ProviderHealth, ProviderHealthMetrics, ProviderStatus};
+pub use vllm_client::VLlmClient;