autoinference-utils 0.2.1__tar.gz → 0.2.2__tar.gz

{autoinference_utils-0.2.1 → autoinference_utils-0.2.2}/.gitignore

@@ -242,5 +242,7 @@ Temporary Items
 benchmark_results/
 docs/*
 !docs/data-model.md
+!docs/dogfood-feedback.md
+!docs/main-architecture-explainer.md
 !docs/recipe-seeding.md
 !docs/sweep-learnings.md

{autoinference_utils-0.2.1 → autoinference_utils-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: autoinference-utils
-Version: 0.2.1
+Version: 0.2.2
 Summary: Shared endpoint abstractions for autoinference deployments
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{autoinference_utils-0.2.1 → autoinference_utils-0.2.2}/autoinference_utils/endpoint.py

@@ -238,9 +238,17 @@ class VLLMEndpoint(Endpoint):
         health_poll_interval: float = 5.0,
         health_request_timeout: float = 5.0,
     ):
-        super().__init__(base_url=f"http://localhost:{worker_port}")
+        # In bench mode vLLM runs on worker_port + 10000 and a /bench proxy
+        # listens on the original worker_port; both fall through to the same
+        # port when bench mode is off.
+        self.bench_mode = os.environ.get(BENCH_MODE_ENV) == "1"
+        self.listen_port = worker_port
+        vllm_port = (
+            worker_port + BENCH_MODE_PORT_OFFSET if self.bench_mode else worker_port
+        )
+        super().__init__(base_url=f"http://localhost:{vllm_port}")
         self.model = model
-        self.worker_port = worker_port
+        self.worker_port = vllm_port
         self.extra_server_args = (
             dict(extra_server_args) if extra_server_args else {}
         )
@@ -248,6 +256,10 @@ class VLLMEndpoint(Endpoint):
         self.health_poll_interval = health_poll_interval
         self.health_request_timeout = health_request_timeout
         self._proc: Optional[subprocess.Popen] = None
+        self._bench_server: Optional[http.server.ThreadingHTTPServer] = None
+
+        if os.environ.get(DUMMY_WEIGHTS_ENV) == "1":
+            self.extra_server_args.setdefault("--load-format", "dummy")
 
     def _build_cmd(self) -> list[str]:
         cmd = [
@@ -264,13 +276,6 @@ class VLLMEndpoint(Endpoint):
         return cmd
 
     def start(self):
-        if (
-            os.environ.get(BENCH_MODE_ENV) == "1"
-            or os.environ.get(DUMMY_WEIGHTS_ENV) == "1"
-        ):
-            raise NotImplementedError(
-                "bench_mode and dummy weights are not supported for VLLMEndpoint"
-            )
         cmd = self._build_cmd()
         print(f"[vllm] starting: {shlex.join(cmd)}")
         self._proc = subprocess.Popen(cmd)
@@ -281,8 +286,16 @@ class VLLMEndpoint(Endpoint):
             poll_interval=self.health_poll_interval,
             request_timeout=self.health_request_timeout,
         )
+        if self.bench_mode:
+            self._bench_server = start_bench_proxy(
+                listen_port=self.listen_port,
+                upstream_port=self.worker_port,
+            )
 
     def stop(self):
+        if self._bench_server is not None:
+            self._bench_server.shutdown()
+            self._bench_server = None
         terminate_process(self._proc)
         self._proc = None
 

{autoinference_utils-0.2.1 → autoinference_utils-0.2.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "autoinference-utils"
-version = "0.2.1"
+version = "0.2.2"
 description = "Shared endpoint abstractions for autoinference deployments"
 readme = "README.md"
 requires-python = ">=3.10"