sie-server 0.7.2__tar.gz → 0.7.3__tar.gz

{sie_server-0.7.2 → sie_server-0.7.3}/Dockerfile.cpu

@@ -101,8 +101,10 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 # shared-deps resolver.
 RUN .venv/bin/python -c "import sie_audio_prep, torch; print(torch.__version__)"
 
-# Register bundle-libs on sys.path. Content is identical across bundles.
-RUN echo "import sys; sys.path.insert(0, '/app/bundle-libs')" \
+# Register bundle-libs as a site directory so dependency-owned .pth files are
+# processed, then move every discovered bundle path ahead of the shared venv so
+# bundle versions shadow it. Content is identical across bundles.
+RUN echo "import site, sys; _sie_bundle_start = len(sys.path); site.addsitedir('/app/bundle-libs'); sys.path[:] = sys.path[_sie_bundle_start:] + sys.path[:_sie_bundle_start]" \
     > /app/.venv/lib/python3.12/site-packages/_sie_bundle.pth
 
 # .venv cleanup + stdlib trim — runs here so /app/.venv reaches its final

{sie_server-0.7.2 → sie_server-0.7.3}/Dockerfile.cuda12

@@ -110,12 +110,12 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 # intentionally not imported here.
 RUN .venv/bin/python -c "import sie_audio_prep, torch; print(torch.__version__)"
 
-# Register bundle-libs on sys.path via a .pth file executed by site.py at
-# interpreter startup. `sys.path.insert(0, ...)` puts bundle-libs at
-# position 0 so bundle-specific versions SHADOW shared venv versions
-# (required for the transformers5 bundle). Content is identical for every
-# bundle, so writing it here keeps the shared venv layer byte-identical.
-RUN echo "import sys; sys.path.insert(0, '/app/bundle-libs')" \
+# Register bundle-libs as a site directory so dependency-owned .pth files are
+# processed at interpreter startup. Then move every discovered bundle path
+# ahead of the shared venv so bundle-specific versions SHADOW it (required for
+# the transformers5 bundle). Content is identical for every bundle, so writing
+# it here keeps the shared venv layer byte-identical.
+RUN echo "import site, sys; _sie_bundle_start = len(sys.path); site.addsitedir('/app/bundle-libs'); sys.path[:] = sys.path[_sie_bundle_start:] + sys.path[:_sie_bundle_start]" \
     > /app/.venv/lib/python3.12/site-packages/_sie_bundle.pth
 
 # Conservative venv cleanup — only things known to be runtime-safe. Keep
@@ -210,6 +210,8 @@ RUN set -eux; \
 # smaller CUDA base runtime; SGLang-family bundles need the devel toolkit
 # because flashinfer/tvm_ffi perform runtime JIT through nvcc on first decode.
 FROM nvidia/cuda:12.4.1-base-ubuntu22.04 AS runtime-default
+# The CTranslate2 bundle uses the CUDA runtime image, not the devel toolchain.
+FROM nvidia/cuda:12.4.1-runtime-ubuntu22.04 AS runtime-ctranslate2
 FROM nvidia/cuda:12.4.1-base-ubuntu22.04 AS runtime-transformers5
 FROM nvidia/cuda:12.9.1-cudnn-devel-ubuntu22.04 AS runtime-sglang
 ENV CUDA_HOME=/usr/local/cuda \

{sie_server-0.7.2 → sie_server-0.7.3}/Dockerfile.cuda13

@@ -1,14 +1,12 @@
 # syntax=docker/dockerfile:1
-# SIE Server - CUDA 13 Image (SGLang cu130 bundles)
+# SIE Server - CUDA 13 Image (engine-isolated cu130 bundles)
 # Build from repo root:
 #   docker build -f packages/sie_server/Dockerfile.cuda13 --build-arg BUNDLE=sglang-cu130 -t sie-server:cuda13-sglang-cu130 .
 #
-# WHY A SEPARATE PLATFORM: the sglang-cu130 bundle needs sglang 0.5.13, which pins
-# torch==2.11.0 (cu130) + transformers 5.8.x and JIT-compiles cu130 kernels
-# (flashinfer / tvm_ffi) through nvcc on first decode — so it requires a CUDA 13
-# devel runtime. Keeping it here (platform ``cuda13``) instead of inside
-# Dockerfile.cuda12 means a ``cuda13-sglang-cu130`` image is honestly labelled CUDA 13,
-# not a CUDA-13 image hiding under a ``cuda12-*`` tag (see #1415).
+# WHY A SEPARATE PLATFORM: CUDA 13 engines carry independent exact Torch,
+# Transformers, FlashInfer, and native-library closures. Keeping them here
+# (platform ``cuda13``) instead of Dockerfile.cuda12 makes image tags honest
+# while still sharing the expensive CUDA 13 base layers.
 #
 # STRUCTURE: mirrors Dockerfile.cuda12's deps→base→builder→runtime stages so the
 # shared logic (standalone Python, editable installs, bundle-libs .pth shadow,
@@ -18,7 +16,7 @@
 # cu130 bundle), so there is no per-bundle runtime indirection.
 #
 # DUAL-TORCH CARRY: the shared base venv still installs torch 2.9 (cu129) from
-# pyproject, and the sglang-cu130 bundle overlays torch 2.11 (cu130) into bundle-libs
+# pyproject, and CUDA 13 bundles overlay torch 2.11 (cu130) into bundle-libs
 # (~+3 GB). Slimming torch out of sie-server core to drop the carry is a tracked
 # follow-up (#1415).
 
@@ -50,13 +48,16 @@ ENV DEBIAN_FRONTEND=noninteractive \
     UV_PYTHON_INSTALL_DIR=/opt/python \
     PIP_DISABLE_PIP_VERSION_CHECK=1
 
-# Minimal apt footprint: just enough for curl + git; no deadsnakes PPA.
+# Minimal apt footprint: curl/git plus the MPI runtime required by
+# TensorRT-LLM's Python import; no OpenMPI development headers are needed.
 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \
     apt-get update && apt-get install -y --no-install-recommends \
     ca-certificates \
     curl \
-    git
+    git \
+    libopenmpi3 \
+    openmpi-bin
 
 ARG UV_VERSION
 RUN curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh \
@@ -117,12 +118,12 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 # Sanity-check shared venv imports.
 RUN .venv/bin/python -c "import sie_audio_prep, torch; print(torch.__version__)"
 
-# Register bundle-libs on sys.path via a .pth executed by site.py at startup.
-# sys.path.insert(0, ...) puts bundle-libs at position 0 so bundle-specific
-# versions SHADOW the shared venv (the cu130 torch/transformers/sglang
-# overlay). Content is identical for every bundle, so writing it here keeps the
-# shared venv layer byte-identical.
-RUN echo "import sys; sys.path.insert(0, '/app/bundle-libs')" \
+# Register bundle-libs as a site directory so dependency-owned .pth files are
+# processed at interpreter startup. Then move every discovered bundle path
+# ahead of the shared venv so bundle-specific versions SHADOW it (the cu130
+# torch/transformers/engine overlay). Content is identical for every bundle, so
+# writing it here keeps the shared venv layer byte-identical.
+RUN echo "import site, sys; _sie_bundle_start = len(sys.path); site.addsitedir('/app/bundle-libs'); sys.path[:] = sys.path[_sie_bundle_start:] + sys.path[:_sie_bundle_start]" \
     > /app/.venv/lib/python3.12/site-packages/_sie_bundle.pth
 
 # Conservative venv cleanup — only runtime-safe removals. Keep torch/include
@@ -155,7 +156,7 @@ RUN .venv/bin/python -m sie_server.cli resolve-deps \
     && cat /tmp/bundle-requirements.txt
 
 # cu130 bundle deps go into a separate site-packages tree (``bundle-libs``,
-# prepended to sys.path via the .pth above). The sglang-cu130 bundle needs torch 2.11
+# prepended to sys.path via the .pth above). CUDA 13 engines need torch 2.11
 # from the cu130 PyTorch index, where the only ``2.11.0`` is the ``+cu130``
 # local-version wheel; pip's multi-index resolution does not reliably prefer it
 # over PyPI's default-CUDA build, so cuda13 bundles resolve with ``uv`` and
@@ -164,7 +165,7 @@ RUN .venv/bin/python -m sie_server.cli resolve-deps \
 # cu130-index resolution was validated with sglang 0.5.13, torch 2.11, and a
 # cu130 runtime in a fresh GPU environment.
 #
-# COVERAGE NOTE: the cuda13 image CI smoke (.github/workflows/sglang-cu130-image.yml)
+# COVERAGE NOTE: the CUDA 13 image CI callers
 # builds this image and asserts on CPU that the bundle-libs cu130 overlay
 # SHADOWS the base venv (import torch -> 2.11+cu130, transformers 5.x). It does
 # NOT exercise GPU init: the cu12 base venv + cu130 bundle-libs both expose the
@@ -220,6 +221,8 @@ ENV DEBIAN_FRONTEND=noninteractive
 # libnuma1: required by sgl_kernel; import fails with a misleading SM-arch error
 #   without it.
 # libgomp1: torch OpenMP runtime.
+# libopenmpi3/openmpi-bin: TensorRT-LLM imports MPI at runtime; headers are not
+#   required by the qualified wheel closure.
 # libspatialindex-c6 / libgl1 / libglib2.0-0 / libice6 / libsm6 / libx11-6 /
 #   libxcb1 / libxext6: docling/opencv dlopen chain (kept symmetric with the
 #   cuda12 runtime so the same model bundle deps import cleanly).
@@ -235,9 +238,11 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     libgomp1 \
     libice6 \
     libnuma1 \
+    libopenmpi3 \
     libsm6 \
     libspatialindex-c6 \
     ninja-build \
+    openmpi-bin \
     libx11-6 \
     libxcb1 \
     libxext6
@@ -278,7 +283,8 @@ LABEL org.opencontainers.image.title="SIE Server" \
       org.opencontainers.image.description="Search Inference Engine - ${BUNDLE} bundle (CUDA 13)" \
       sie.bundle="${BUNDLE}"
 
-ENV PATH="/app/.venv/bin:$PATH" \
+# ``uv --target`` writes engine entrypoints beside the package overlay.
+ENV PATH="/app/bundle-libs/bin:/app/.venv/bin:$PATH" \
     PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1 \
     HF_HOME=/app/.cache/huggingface \

{sie_server-0.7.2 → sie_server-0.7.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: sie-server
-Version: 0.7.2
+Version: 0.7.3
 Summary: Search Inference Engine - GPU inference server for search workloads
 License: Apache-2.0
 License-File: LICENSE

sie_server-0.7.3/bundles/ctranslate2.yaml

@@ -0,0 +1,6 @@
+name: ctranslate2
+priority: 19
+adapters:
+- sie_server.adapters.ctranslate2.generation
+deps:
+  ctranslate2: '==4.8.1'

sie_server-0.7.3/bundles/tensorrt-llm.yaml

@@ -0,0 +1,37 @@
+name: tensorrt-llm
+priority: 20
+default: false
+# Reusable direct-HF encoder-decoder runtime. This is a CUDA 13 engine lane,
+# not a checkpoint-specific bundle; compatible T5/BART/mBART-family profiles
+# opt in through the generic adapter module below.
+platform: cuda13
+adapters:
+- sie_server.adapters.tensorrt_llm.generation
+deps:
+  # Exact prerelease qualified from tag commit
+  # 1cef02e901be43081b1ba6d4981e94ed3bd9c1e8. Compatibility overlays verify
+  # the complete rc24 source files they patch and refuse any other build.
+  tensorrt-llm: '==1.3.0rc24'
+  # rc24's direct-HF T5 closure. Torch resolves to the +cu130 wheel through
+  # Dockerfile.cuda13's explicit PyTorch index.
+  torch: '==2.11.0'
+  transformers: '==5.5.4'
+  flashinfer-python: '==0.6.16'
+  triton: '==3.6.0'
+  nvidia-cutlass-dsl: '==4.5.0'
+  nvidia-cuda-tileiras: '==13.1.80'
+  nvidia-matmul-heuristics: '==0.1.0.27'
+  # cuda-python 13.3.1 permits cuda-pathfinder~=1.1, while cuda-core and
+  # cuda-bindings only require >=1.4.2. Pin the import-proven rc24 closure so
+  # a newer transitive release cannot silently change the engine image.
+  cuda-pathfinder: '==1.7.0'
+  # rc24 permits an NCCL range; lock the import-proven cu13 build so PyPI and
+  # the cu130 indexes cannot choose a different ABI-compatible-looking wheel.
+  nvidia-nccl-cu13: '==2.28.9'
+  flash-attn-4: '==4.0.0b11'
+  torch-c-dlpack-ext: '==0.1.3'
+  xgrammar: '==0.1.32'
+  apache-tvm-ffi: '==0.1.6'
+  # The published rc24 closure leaves mpmath open; a newer resolution breaks
+  # native import. 1.3.0 is the exact import-proven value.
+  mpmath: '==1.3.0'

sie_server-0.7.3/models/google__madlad400-3b-mt.yaml

@@ -0,0 +1,42 @@
+sie_id: "google/madlad400-3b-mt"
+hf_id: "google/madlad400-3b-mt"
+hf_revision: fa184c675da0b5c9e1c8694fccd4e12e2d422094
+inputs:
+  text: true
+  image: false
+  audio: false
+  video: false
+tasks:
+  encode: null
+  score: null
+  extract: null
+  generate:
+    context_length: 512
+    max_output_tokens: 512
+    capabilities:
+      grammar: []
+      streaming: true
+      tools: false
+    chat_template_kwargs: {}
+max_sequence_length: 512
+profiles:
+  default:
+    max_batch_tokens: 4096
+    kv_budget_tokens: 4096
+    compute_precision: float32
+    adapter_path: sie_server.adapters.ctranslate2.generation:CTranslate2GenerationAdapter
+    adapter_options:
+      loadtime:
+        serving_artifact:
+          format: ctranslate2
+          repo_id: superlinked/madlad400-3b-mt-ctranslate2-float32
+          revision: 2cac811471ffb01ea6502ab4d317d5a7e9cb6f7d
+          manifest_path: sie-serving-artifact.json
+          manifest_sha256: 7583a4fbe2e51551dc61644983b305d22e3c54ec9e3c553d0030b68e70615768
+          compute_type: float32
+        native_batch_max_size: 8
+        native_batch_max_tokens: 4096
+      runtime:
+        first_chunk_timeout_s: 180
+        inter_chunk_timeout_s: 10
+        overall_timeout_s: 600

{sie_server-0.7.2 → sie_server-0.7.3}/openapi.json

@@ -3,7 +3,7 @@
   "info": {
     "title": "SIE Server",
     "description": "Search Inference Engine - GPU inference server for search workloads",
-    "version": "0.7.2"
+    "version": "0.7.3"
   },
   "paths": {
     "/healthz": {
@@ -1428,6 +1428,11 @@
       },
       "ModelCapabilities": {
         "properties": {
+          "streaming": {
+            "type": "boolean",
+            "title": "Streaming",
+            "default": true
+          },
           "grammar": {
             "items": {
               "type": "string"
@@ -2985,6 +2990,34 @@
             "description": "Client-safe error message",
             "title": "Message",
             "type": "string"
+          },
+          "param": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "default": null,
+            "description": "Exact unsupported request field, when applicable",
+            "title": "Param"
+          },
+          "retry_after_s": {
+            "anyOf": [
+              {
+                "maximum": 60,
+                "minimum": 1,
+                "type": "integer"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "default": null,
+            "description": "Authoritative retry hint in seconds for RESOURCE_EXHAUSTED only; null for other terminal errors",
+            "title": "Retry After S"
           }
         },
         "required": [

{sie_server-0.7.2 → sie_server-0.7.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sie-server"
-version = "0.7.2"
+version = "0.7.3"
 description = "Search Inference Engine - GPU inference server for search workloads"
 requires-python = ">=3.12,<3.13"
 license = { text = "Apache-2.0" }