autoinference-utils 0.2.5__tar.gz → 0.2.6__tar.gz

{autoinference_utils-0.2.5 → autoinference_utils-0.2.6}/.gitignore

@@ -12,6 +12,7 @@ node_modules/
 build/
 develop-eggs/
 dist/
+!.github/actions/paths-filter/dist/
 downloads/
 eggs/
 .eggs/
@@ -151,6 +152,7 @@ activemq-data/
 .env
 .envrc
 .venv
+.autoinference/
 env/
 venv/
 ENV/
@@ -239,10 +241,6 @@ Network Trash Folder
 Temporary Items
 .apdisk
 
+# Ephemeral files from Experiments
 benchmark_results/
-docs/*
-!docs/data-model.md
-!docs/dogfood-feedback.md
-!docs/main-architecture-explainer.md
-!docs/recipe-seeding.md
-!docs/sweep-learnings.md
+src/autoinference/deployments/

{autoinference_utils-0.2.5 → autoinference_utils-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.5
 Name: autoinference-utils
-Version: 0.2.5
+Version: 0.2.6
 Summary: Shared endpoint abstractions for autoinference deployments
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{autoinference_utils-0.2.5 → autoinference_utils-0.2.6}/autoinference_utils/endpoint.py

@@ -30,12 +30,55 @@ BENCH_MODE_PORT_OFFSET = 10000
 MODAL_FLASH_REQUEST_UUID_HEADER= "modal-flash-request-uuid"
 MODAL_SESSION_ID_HEADER = "modal-session-id"
 
-# Endpoint class names whose servers emit OAI streaming chunks that
-# sglang.bench_serving can't parse. deploy_and_bench auto-injects
-# --openai-stream-compat for deployments importing any of these.
 ENDPOINTS_REQUIRING_OAI_STREAM_COMPAT: tuple[str, ...] = ("TRTLLMEndpoint",)
 
 
+def server_arg_tokens(flag: str, value: str) -> list[str]:
+    """Render one server arg as argv tokens.
+
+    A value splits on whitespace, so a repeatable flag takes its values as one
+    entry. A lone value starting with `-` keeps the attached `--flag=value`
+    spelling, the only one an argparse-style parser does not read as a flag.
+    """
+    if value and "=" in flag:
+        raise ValueError(f"server arg {flag!r} already carries a value: {value!r}")
+    if value == "":
+        return [flag]
+    tokens = value.split()
+    if len(tokens) == 1 and tokens[0].startswith("-"):
+        return [f"{flag}={tokens[0]}"]
+    return [flag, *tokens]
+
+
+def _flag_name(key: str) -> str:
+    """Return the flag a key sets, whose value may be attached as `--flag=value`."""
+    return key.partition("=")[0]
+
+
+def get_server_arg(
+    server_args: Mapping[str, str], flag: str
+) -> tuple[str, str] | None:
+    """Return the key and value setting a flag under either spelling."""
+    for key, value in server_args.items():
+        if _flag_name(key) == flag:
+            return key, value
+    return None
+
+
+def has_server_arg(server_args: Mapping[str, str], flag: str) -> bool:
+    """Report whether a flag is set under either spelling, plain or attached."""
+    return get_server_arg(server_args, flag) is not None
+
+
+def _merge_server_args(
+    defaults: Mapping[str, str], overrides: Mapping[str, str]
+) -> dict[str, str]:
+    """Overlay overrides on defaults by flag name, so an attached spelling wins."""
+    merged = {_flag_name(key): (key, value) for key, value in defaults.items()}
+    merged.update((_flag_name(key), (key, value)) for key, value in overrides.items())
+    return dict(merged.values())
+
+
 class Endpoint(ABC):
     """A thing with a URL that you can start and stop."""
 
@@ -147,21 +190,27 @@ class SGLangEndpoint(Endpoint):
         if os.environ.get(DUMMY_WEIGHTS_ENV) == "1":
             if (
                 self.load_format is None
-                and "--load-format" not in self.extra_server_args
+                and not has_server_arg(self.extra_server_args, "--load-format")
             ):
                 self.load_format = "dummy"
-            self.extra_server_args.setdefault("--model-loader-extra-config", "{}")
-        
+            if not has_server_arg(self.extra_server_args, "--model-loader-extra-config"):
+                self.extra_server_args["--model-loader-extra-config"] = "{}"
+
         # Request logging enabled, log only metadata by default
         if self.log_requests_level >= 0:
-            # Enable request logging    
-            if "--log-requests" not in self.extra_server_args:
-                self.extra_server_args.setdefault("--log-requests", "")
+            # Enable request logging
+            if not has_server_arg(self.extra_server_args, "--log-requests"):
+                self.extra_server_args["--log-requests"] = ""
             # Set request logging level
-            if "--log-requests-level" not in self.extra_server_args:
-                self.extra_server_args.setdefault("--log-requests-level", str(self.log_requests_level))
+            level = get_server_arg(self.extra_server_args, "--log-requests-level")
+            if level is None:
+                self.extra_server_args["--log-requests-level"] = str(self.log_requests_level)
             else:
-                print(f"[endpoint] --log-requests-level already set to {self.extra_server_args['--log-requests-level']} in server args")
+                key, value = level
+                print(
+                    "[endpoint] --log-requests-level already set in server args: "
+                    f"{value or key}"
+                )
 
             # Add Modal-forwarded request headers to SGLang logged headers
             sglang_log_request_headers = [
@@ -221,12 +270,9 @@ class SGLangEndpoint(Endpoint):
                 ]
             )
 
-        merged = {**self.DEFAULT_OPERATIONAL_ARGS, **self.extra_server_args}
+        merged = _merge_server_args(self.DEFAULT_OPERATIONAL_ARGS, self.extra_server_args)
         for key, value in merged.items():
-            if value == "":
-                cmd.append(key)
-            else:
-                cmd.extend([key, *value.split()])
+            cmd.extend(server_arg_tokens(key, value))
 
         return cmd
 
@@ -297,7 +343,8 @@ class VLLMEndpoint(Endpoint):
         self._bench_server: Optional[http.server.ThreadingHTTPServer] = None
 
         if os.environ.get(DUMMY_WEIGHTS_ENV) == "1":
-            self.extra_server_args.setdefault("--load-format", "dummy")
+            if not has_server_arg(self.extra_server_args, "--load-format"):
+                self.extra_server_args["--load-format"] = "dummy"
 
     def _build_cmd(self) -> list[str]:
         cmd = [
@@ -307,10 +354,7 @@ class VLLMEndpoint(Endpoint):
             "--model", self.model,
         ]
         for key, value in self.extra_server_args.items():
-            if value == "":
-                cmd.append(key)
-            else:
-                cmd.extend([key, *value.split()])
+            cmd.extend(server_arg_tokens(key, value))
         return cmd
 
     def start(self):
@@ -377,10 +421,7 @@ class TRTLLMEndpoint(Endpoint):
             "--port", str(self.worker_port),
         ]
         for key, value in self.extra_server_args.items():
-            if value == "":
-                cmd.append(key)
-            else:
-                cmd.extend([key, *value.split()])
+            cmd.extend(server_arg_tokens(key, value))
         return cmd
 
     def health_check(self) -> str | None:

{autoinference_utils-0.2.5 → autoinference_utils-0.2.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "autoinference-utils"
-version = "0.2.5"
+version = "0.2.6"
 description = "Shared endpoint abstractions for autoinference deployments"
 readme = "README.md"
 requires-python = ">=3.10"