autoinference-utils 0.2.2__tar.gz → 0.2.3__tar.gz

{autoinference_utils-0.2.2 → autoinference_utils-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: autoinference-utils
-Version: 0.2.2
+Version: 0.2.3
 Summary: Shared endpoint abstractions for autoinference deployments
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{autoinference_utils-0.2.2 → autoinference_utils-0.2.3}/autoinference_utils/endpoint.py

@@ -26,6 +26,9 @@ BENCH_MODE_ENV = "ENDPOINT_BENCH_MODE"
 DUMMY_WEIGHTS_ENV = "ENDPOINT_DUMMY"
 BENCH_MODE_PORT_OFFSET = 10000
 
+MODAL_FLASH_REQUEST_UUID_HEADER= "x-modal-flash-request-uuid"
+MODAL_SESSION_ID_HEADER = "modal-session-id"
+
 # Endpoint class names whose servers emit OAI streaming chunks that
 # sglang.bench_serving can't parse. deploy_and_bench auto-injects
 # --openai-stream-compat for deployments importing any of these.
@@ -93,6 +96,7 @@ class SGLangEndpoint(Endpoint):
         health_timeout: float = 20 * 60,
         health_poll_interval: float = 5.0,
         health_request_timeout: float = 5.0,
+        log_requests_level: int = 0,
     ):
         # In bench mode SGLang runs on worker_port + 10000 and a /bench proxy
         # listens on the original worker_port; both fall through to the same
@@ -124,6 +128,13 @@ class SGLangEndpoint(Endpoint):
         self.health_timeout = health_timeout
         self.health_poll_interval = health_poll_interval
         self.health_request_timeout = health_request_timeout
+
+        # SGLang Log request level: -1 = disabled, 0 = metadata only, 1,2,3 in increasing verbosity
+        # Refer to https://docs.sglang.io/docs/advanced_features/server_arguments#logging for more details
+        if not (log_requests_level >= -1 and log_requests_level <= 3):
+            raise ValueError("log_requests_level must be between -1 and 3")
+        self.log_requests_level = log_requests_level
+
         self._proc: Optional[subprocess.Popen] = None
         self._bench_server: Optional[http.server.ThreadingHTTPServer] = None
 
@@ -139,6 +150,32 @@ class SGLangEndpoint(Endpoint):
             ):
                 self.load_format = "dummy"
             self.extra_server_args.setdefault("--model-loader-extra-config", "{}")
+        
+        # Request logging enabled, log only metadata by default
+        if self.log_requests_level >= 0:
+            # Enable request logging    
+            if "--log-requests" not in self.extra_server_args:
+                self.extra_server_args.setdefault("--log-requests", "")
+            # Set request logging level
+            if "--log-requests-level" not in self.extra_server_args:
+                self.extra_server_args.setdefault("--log-requests-level", str(self.log_requests_level))
+            else:
+                print(f"[endpoint] --log-requests-level already set to {self.extra_server_args['--log-requests-level']} in server args")
+
+            # Add Modal-forwarded request headers to SGLang logged headers
+            sglang_log_request_headers = [
+                h.strip().lower()
+                for h in os.environ.get("SGLANG_LOG_REQUEST_HEADERS", "").split(",")
+                if h.strip()
+            ]
+            if MODAL_FLASH_REQUEST_UUID_HEADER not in sglang_log_request_headers:
+                sglang_log_request_headers.append(MODAL_FLASH_REQUEST_UUID_HEADER)
+            if MODAL_SESSION_ID_HEADER not in sglang_log_request_headers:
+                sglang_log_request_headers.append(MODAL_SESSION_ID_HEADER)
+
+            self.log_request_headers = ",".join(sglang_log_request_headers)
+            os.environ["SGLANG_LOG_REQUEST_HEADERS"] = self.log_request_headers
+
 
     def _build_cmd(self) -> list[str]:
         cmd = [

{autoinference_utils-0.2.2 → autoinference_utils-0.2.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "autoinference-utils"
-version = "0.2.2"
+version = "0.2.3"
 description = "Shared endpoint abstractions for autoinference deployments"
 readme = "README.md"
 requires-python = ">=3.10"