autoinference-utils 0.2.1__tar.gz → 0.2.3__tar.gz

{autoinference_utils-0.2.1 → autoinference_utils-0.2.3}/.gitignore

@@ -242,5 +242,7 @@ Temporary Items
 benchmark_results/
 docs/*
 !docs/data-model.md
+!docs/dogfood-feedback.md
+!docs/main-architecture-explainer.md
 !docs/recipe-seeding.md
 !docs/sweep-learnings.md

{autoinference_utils-0.2.1 → autoinference_utils-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: autoinference-utils
-Version: 0.2.1
+Version: 0.2.3
 Summary: Shared endpoint abstractions for autoinference deployments
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{autoinference_utils-0.2.1 → autoinference_utils-0.2.3}/autoinference_utils/endpoint.py

@@ -26,6 +26,9 @@ BENCH_MODE_ENV = "ENDPOINT_BENCH_MODE"
 DUMMY_WEIGHTS_ENV = "ENDPOINT_DUMMY"
 BENCH_MODE_PORT_OFFSET = 10000
 
+MODAL_FLASH_REQUEST_UUID_HEADER= "x-modal-flash-request-uuid"
+MODAL_SESSION_ID_HEADER = "modal-session-id"
+
 # Endpoint class names whose servers emit OAI streaming chunks that
 # sglang.bench_serving can't parse. deploy_and_bench auto-injects
 # --openai-stream-compat for deployments importing any of these.
@@ -93,6 +96,7 @@ class SGLangEndpoint(Endpoint):
         health_timeout: float = 20 * 60,
         health_poll_interval: float = 5.0,
         health_request_timeout: float = 5.0,
+        log_requests_level: int = 0,
     ):
         # In bench mode SGLang runs on worker_port + 10000 and a /bench proxy
         # listens on the original worker_port; both fall through to the same
@@ -124,6 +128,13 @@ class SGLangEndpoint(Endpoint):
         self.health_timeout = health_timeout
         self.health_poll_interval = health_poll_interval
         self.health_request_timeout = health_request_timeout
+
+        # SGLang Log request level: -1 = disabled, 0 = metadata only, 1,2,3 in increasing verbosity
+        # Refer to https://docs.sglang.io/docs/advanced_features/server_arguments#logging for more details
+        if not (log_requests_level >= -1 and log_requests_level <= 3):
+            raise ValueError("log_requests_level must be between -1 and 3")
+        self.log_requests_level = log_requests_level
+
         self._proc: Optional[subprocess.Popen] = None
         self._bench_server: Optional[http.server.ThreadingHTTPServer] = None
 
@@ -139,6 +150,32 @@ class SGLangEndpoint(Endpoint):
             ):
                 self.load_format = "dummy"
             self.extra_server_args.setdefault("--model-loader-extra-config", "{}")
+        
+        # Request logging enabled, log only metadata by default
+        if self.log_requests_level >= 0:
+            # Enable request logging    
+            if "--log-requests" not in self.extra_server_args:
+                self.extra_server_args.setdefault("--log-requests", "")
+            # Set request logging level
+            if "--log-requests-level" not in self.extra_server_args:
+                self.extra_server_args.setdefault("--log-requests-level", str(self.log_requests_level))
+            else:
+                print(f"[endpoint] --log-requests-level already set to {self.extra_server_args['--log-requests-level']} in server args")
+
+            # Add Modal-forwarded request headers to SGLang logged headers
+            sglang_log_request_headers = [
+                h.strip().lower()
+                for h in os.environ.get("SGLANG_LOG_REQUEST_HEADERS", "").split(",")
+                if h.strip()
+            ]
+            if MODAL_FLASH_REQUEST_UUID_HEADER not in sglang_log_request_headers:
+                sglang_log_request_headers.append(MODAL_FLASH_REQUEST_UUID_HEADER)
+            if MODAL_SESSION_ID_HEADER not in sglang_log_request_headers:
+                sglang_log_request_headers.append(MODAL_SESSION_ID_HEADER)
+
+            self.log_request_headers = ",".join(sglang_log_request_headers)
+            os.environ["SGLANG_LOG_REQUEST_HEADERS"] = self.log_request_headers
+
 
     def _build_cmd(self) -> list[str]:
         cmd = [
@@ -238,9 +275,17 @@ class VLLMEndpoint(Endpoint):
         health_poll_interval: float = 5.0,
         health_request_timeout: float = 5.0,
     ):
-        super().__init__(base_url=f"http://localhost:{worker_port}")
+        # In bench mode vLLM runs on worker_port + 10000 and a /bench proxy
+        # listens on the original worker_port; both fall through to the same
+        # port when bench mode is off.
+        self.bench_mode = os.environ.get(BENCH_MODE_ENV) == "1"
+        self.listen_port = worker_port
+        vllm_port = (
+            worker_port + BENCH_MODE_PORT_OFFSET if self.bench_mode else worker_port
+        )
+        super().__init__(base_url=f"http://localhost:{vllm_port}")
         self.model = model
-        self.worker_port = worker_port
+        self.worker_port = vllm_port
         self.extra_server_args = (
             dict(extra_server_args) if extra_server_args else {}
         )
@@ -248,6 +293,10 @@ class VLLMEndpoint(Endpoint):
         self.health_poll_interval = health_poll_interval
         self.health_request_timeout = health_request_timeout
         self._proc: Optional[subprocess.Popen] = None
+        self._bench_server: Optional[http.server.ThreadingHTTPServer] = None
+
+        if os.environ.get(DUMMY_WEIGHTS_ENV) == "1":
+            self.extra_server_args.setdefault("--load-format", "dummy")
 
     def _build_cmd(self) -> list[str]:
         cmd = [
@@ -264,13 +313,6 @@ class VLLMEndpoint(Endpoint):
         return cmd
 
     def start(self):
-        if (
-            os.environ.get(BENCH_MODE_ENV) == "1"
-            or os.environ.get(DUMMY_WEIGHTS_ENV) == "1"
-        ):
-            raise NotImplementedError(
-                "bench_mode and dummy weights are not supported for VLLMEndpoint"
-            )
         cmd = self._build_cmd()
         print(f"[vllm] starting: {shlex.join(cmd)}")
         self._proc = subprocess.Popen(cmd)
@@ -281,8 +323,16 @@ class VLLMEndpoint(Endpoint):
             poll_interval=self.health_poll_interval,
             request_timeout=self.health_request_timeout,
         )
+        if self.bench_mode:
+            self._bench_server = start_bench_proxy(
+                listen_port=self.listen_port,
+                upstream_port=self.worker_port,
+            )
 
     def stop(self):
+        if self._bench_server is not None:
+            self._bench_server.shutdown()
+            self._bench_server = None
         terminate_process(self._proc)
         self._proc = None
 

{autoinference_utils-0.2.1 → autoinference_utils-0.2.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "autoinference-utils"
-version = "0.2.1"
+version = "0.2.3"
 description = "Shared endpoint abstractions for autoinference deployments"
 readme = "README.md"
 requires-python = ">=3.10"