konduktor-nightly 0.1.0.dev20250919104536__py3-none-any.whl → 0.1.0.dev20250921104307__py3-none-any.whl

konduktor/cli.py

@@ -973,7 +973,9 @@ def down(
 
     if all:
         assert jobs_specs is not None, f'No jobs found in namespace {namespace}'
-        assert len(jobs_specs) > 0, f'No jobs found in namespace {namespace}'
+        if len(jobs_specs) == 0:
+            click.secho(f'No jobs found in namespace {namespace}', fg='yellow')
+            return
         jobs = [job['metadata']['name'] for job in jobs_specs]
     elif jobs:
         # Get all available jobs to match against patterns
@@ -1630,7 +1632,7 @@ def list_secrets(all_users: bool):
 
 @cli.group(cls=_NaturalOrderGroup)
 def serve():
-    """Manage LLM serving with Konduktor.
+    """Manage deployment serving with Konduktor.
 
     USAGE: konduktor serve COMMAND
 

konduktor/manifests/aibrix-setup.yaml

@@ -0,0 +1,274 @@
+# Aibrix Setup - vLLM Deployment Infrastructure
+# 
+# This file sets up the infrastructure needed for vLLM (Aibrix) deployments:
+# 1. Envoy Gateway configuration for HTTP routing
+# 2. Aibrix Activator service for request-based autoscaling (KPA)
+# 3. HTTP route mirroring for prewarming vLLM models
+# 4. Lua script for extracting model names from OpenAI-compatible requests
+#
+# The activator tracks incoming requests and provides metrics to scale
+# vLLM deployments based on demand (requests per second).
+
+# This file is kept separate from apoxy setup files because it is 
+# only used in actual clusters, not in the test kind clusters.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: envoy-gateway-config
+  namespace: envoy-gateway-system
+data:
+  envoy-gateway.yaml: |
+    apiVersion: gateway.envoyproxy.io/v1alpha1
+    kind: EnvoyGateway
+    provider:
+      type: Kubernetes
+    gateway:
+      controllerName: gateway.envoyproxy.io/gatewayclass-controller
+    extensionApis:
+      enableEnvoyPatchPolicy: true
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: aibrix-activator
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: activator-code
+  namespace: aibrix-activator
+data:
+  activator.py: |
+    import os, time, json
+    from collections import defaultdict, deque
+    from fastapi import FastAPI, Request
+    from fastapi.responses import PlainTextResponse, JSONResponse
+
+    NAMESPACE = os.getenv("NAMESPACE", "default")
+    WINDOW_SEC = int(os.getenv("WINDOW_SEC", "30"))        # demand lookback
+    CAPACITY_RPS = float(os.getenv("CAPACITY_RPS", "1.0")) # per-replica capacity
+    MIN_WAKE = int(os.getenv("MIN_REPLICA_ON_WAKE", "1"))
+    MAX_REPLICAS = int(os.getenv("MAX_REPLICAS", "8"))
+
+    app = FastAPI()
+    events = defaultdict(deque)  # key=(ns,model) -> deque[timestamps]
+
+    def _prune(q, now):
+        while q and now - q[0] > WINDOW_SEC: q.popleft()
+
+    def _bump(ns, model):
+        now = time.time()
+        q = events[(ns, model)]
+        q.append(now)
+        _prune(q, now)
+
+    def _desired(ns, model):
+        now = time.time()
+        q = events[(ns, model)]
+        _prune(q, now)
+        rps = len(q) / max(WINDOW_SEC, 1)
+        if len(q) == 0: return 0
+        # Convert demand to desired replicas
+        import math
+        d = max(MIN_WAKE, math.ceil(rps / max(CAPACITY_RPS, 1e-6)))
+        return max(0, min(d, MAX_REPLICAS))
+
+    def _extract_model(headers, body_bytes):
+        # Prefer header (OpenAI-compatible)
+        m = headers.get("model") or headers.get("x-model")
+        if m: return m
+        # Try JSON body
+        try:
+            j = json.loads(body_bytes or b"{}")
+            if isinstance(j, dict):
+                # OpenAI schema: {"model": "...", ...}
+                if "model" in j and isinstance(j["model"], str):
+                    return j["model"]
+        except Exception:
+            pass
+        return None
+
+    # Mirror endpoints (same as your API paths); quick 204 response
+    @app.post("/v1/completions")
+    @app.post("/v1/chat/completions")
+    async def mirrored(request: Request):
+        body = await request.body()
+        model = _extract_model(request.headers, body)
+        if model:
+            _bump(NAMESPACE, model)
+        return JSONResponse({"ok": True}, status_code=204)
+
+    # Catch-all POST (safety net if your gateway uses different paths)
+    @app.post("/{full_path:path}")
+    async def mirrored_generic(request: Request, full_path: str):
+        body = await request.body()
+        model = _extract_model(request.headers, body)
+        if model:
+            _bump(NAMESPACE, model)
+        return JSONResponse({"ok": True}, status_code=204)
+
+    # Metrics for KPA and Debugging
+    @app.get("/metrics/{ns}/{model}", response_class=PlainTextResponse)
+    async def metrics(ns: str, model: str):
+        d = _desired(ns, model)
+        now = time.time()
+        q = events[(ns, model)]
+        _prune(q, now)
+        rps = len(q) / max(WINDOW_SEC, 1)
+        return (
+            "# HELP vllm:deployment_replicas Number of suggested replicas.\n"
+            "# TYPE vllm:deployment_replicas gauge\n"
+            f'vllm:deployment_replicas{{namespace="{ns}",model_name="{model}"}} {d}\n'
+            "# HELP vllm:observed_rps Incoming requests per second.\n"
+            "# TYPE vllm:observed_rps gauge\n"
+            f'vllm:observed_rps{{namespace="{ns}",model_name="{model}"}} {rps:.2f}\n'
+        )
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: aibrix-activator
+  namespace: aibrix-activator
+spec:
+  replicas: 1
+  selector: { matchLabels: { app: aibrix-activator } }
+  template:
+    metadata: { labels: { app: aibrix-activator } }
+    spec:
+      containers:
+      - name: activator
+        image: python:3.11-slim
+        command: ["bash","-lc"]
+        args:
+          - |
+            pip install fastapi uvicorn >/dev/null && \
+            uvicorn activator:app --host 0.0.0.0 --port 8080
+        env:
+        - { name: NAMESPACE, value: "default" }
+        - { name: WINDOW_SEC, value: "30" }
+        - { name: CAPACITY_RPS, value: "1.0" }
+        - { name: MIN_REPLICA_ON_WAKE, value: "1" }
+        - { name: MAX_REPLICAS, value: "8" }
+        ports: [{containerPort: 8080}]
+        volumeMounts:
+        - { name: code, mountPath: /app/activator.py, subPath: activator.py }
+        workingDir: /app
+      volumes:
+      - name: code
+        configMap: { name: activator-code }
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: aibrix-activator
+  namespace: aibrix-activator
+spec:
+  selector: { app: aibrix-activator }
+  ports:
+  - name: http
+    port: 8080
+    targetPort: 8080
+    protocol: TCP
+  type: ClusterIP
+---
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: ReferenceGrant
+metadata:
+  name: allow-httproute-to-activator
+  namespace: aibrix-activator
+spec:
+  from:
+  - group: gateway.networking.k8s.io
+    kind: HTTPRoute
+    namespace: aibrix-system
+  to:
+  - group: ""
+    kind: Service
+    name: aibrix-activator
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: activator-mirror-sink
+  namespace: aibrix-system
+spec:
+  parentRefs:
+  - group: gateway.networking.k8s.io
+    kind: Gateway
+    name: aibrix-eg
+    namespace: aibrix-system
+  rules:
+  - matches:
+    - path:
+        type: PathPrefix
+        value: /__activator_sink__
+    backendRefs:
+    - name: aibrix-activator
+      namespace: aibrix-activator
+      port: 8080
+---
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: EnvoyPatchPolicy
+metadata:
+  name: prewarm-completions-lua
+  namespace: aibrix-system
+spec:
+  targetRef:
+    group: gateway.networking.k8s.io
+    kind: Gateway
+    name: aibrix-eg
+  type: JSONPatch
+  jsonPatches:
+    - type: "type.googleapis.com/envoy.config.listener.v3.Listener"
+      name: "aibrix-system/aibrix-eg/http"
+      operation:
+        op: add
+        path: "/default_filter_chain/filters/0/typed_config/http_filters/0"
+        value:
+          name: envoy.filters.http.lua
+          typed_config:
+            "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua
+            inlineCode: |
+              function envoy_on_request(handle)
+                local path = handle:headers():get(":path") or ""
+                if string.find(path, "^/v1/completions") or string.find(path, "^/v1/chat/completions") then
+                  -- Try to get model from header first
+                  local model = handle:headers():get("model") or ""
+                  
+                  -- If no model in header, try to extract from JSON body
+                  if model == "" then
+                    local ct = handle:headers():get("content-type") or ""
+                    if string.find(ct:lower(), "application/json") then
+                      local body = handle:body()
+                      if body and body:length() > 0 then
+                        local raw = body:getBytes(0, math.min(body:length(), 1024))
+                        -- Simple regex to extract model from JSON: "model":"value"
+                        local model_match = raw:match('"model"%s*:%s*"([^"]+)"')
+                        if model_match then
+                          model = model_match
+                        end
+                      end
+                    end
+                  end
+                  
+                  -- Only proceed if we have a model
+                  if model ~= "" then
+                    -- fire-and-forget wake signal; very short timeout
+                    pcall(function()
+                      handle:httpCall(
+                        "httproute/aibrix-system/activator-mirror-sink/rule/0",
+                        {
+                          [":method"] = "POST",
+                          [":path"] = "/v1/completions",
+                          [":authority"] = "aibrix-activator.aibrix-activator.svc.cluster.local",
+                          ["content-type"] = "application/json",
+                          ["model"] = model
+                        },
+                        "{}",
+                        5 -- ms
+                      )
+                    end)
+                  end
+                end
+              end

konduktor/manifests/apoxy-setup.yaml

@@ -1,8 +1,25 @@
+# Apoxy Setup (Part 1/3) - Core Infrastructure
+#
+# This file sets up the core Apoxy infrastructure for external access to deployments:
+# 1. Apoxy system namespace and RBAC
+# 2. Kubeconfig secret for cluster access (populated by CI)
+# 3. Apoxy tunnel controller and proxy services
+# 4. Network policies for cross-namespace access
+#
+# Split into 2 files because:
+# - apoxy-setup.yaml: Core infrastructure (1 per cluster) (needs to be applied first)
+# - apoxy-setup2.yaml: All routing rules for both deployment types
+
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: apoxy-system
+---
 apiVersion: v1
 kind: Secret
 metadata:
   name: trainy-kubeconfig
-  namespace: default
+  namespace: apoxy-system
 type: Opaque
 data:
   # this gets replaced by buildkite CI secret APOXY_AUTH
@@ -13,7 +30,7 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: kube-controller
-  namespace: default
+  namespace: apoxy-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -35,13 +52,13 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: kube-controller
-  namespace: default
+  namespace: apoxy-system
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: kube-controller
-  namespace: default
+  namespace: apoxy-system
   labels:
     app: kube-controller
 spec:
@@ -85,7 +102,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: kube-controller
-  namespace: default
+  namespace: apoxy-system
   labels:
     app: kube-controller
 spec:
@@ -101,7 +118,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: apoxy-config
-  namespace: default
+  namespace: apoxy-system
 data:
   config.yaml: |
     apiVersion: config.apoxy.dev/v1alpha1
@@ -118,7 +135,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: apoxy
-  namespace: default
+  namespace: apoxy-system
   labels:
     app: apoxy
 spec:
@@ -133,7 +150,7 @@ spec:
     spec:
       containers:
       - name: apoxy
-        image: apoxy/apoxy:v0.11.14
+        image: apoxy/apoxy:v0.11.18
         command: ["apoxy", "tunnel", "run", "UNIQUE-TEMPNAME", "--insecure-skip-verify"]
         volumeMounts:
         - name: kubeconfig-volume
@@ -148,4 +165,20 @@ spec:
           secretName: trainy-kubeconfig
       - name: apoxy-config-volume
         configMap:
-          name: apoxy-config
+          name: apoxy-config
+---
+# NetworkPolicy to allow Apoxy to reach services in other namespaces
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: apoxy-cross-namespace-access
+  namespace: apoxy-system
+spec:
+  podSelector:
+    matchLabels:
+      app: apoxy
+  policyTypes:
+  - Egress
+  egress:
+  # Allow all egress traffic
+  - {}

konduktor/manifests/apoxy-setup2.yaml

@@ -1,3 +1,20 @@
+# Apoxy Setup (Part 2/2) - Deployment Routing
+#
+# This file sets up Apoxy routing for both vLLM and general deployments:
+# 1. TunnelNode for secure tunnel connection
+# 2. Backend for vLLM pointing to Envoy Gateway
+# 3. HTTPRoute for company.trainy.us -> vLLM deployments
+# 4. Backend for general deployments pointing to nginx ingress
+# 5. HTTPRoute for company2.trainy.us -> general deployments
+# 6. KEDA proxy service for HTTP autoscaling
+# 7. 60s timeout for all requests
+#
+# Split into 2 files because:
+# - apoxy-setup.yaml: Core infrastructure (1 per cluster) (needs to be applied first)
+# - apoxy-setup2.yaml: All routing rules for both deployment types
+
+# NOTE: TunnelNode should technically be in the first apoxy-setup.yaml but it
+# needs to be created after the core infrastructure is created, so we put it here.
 apiVersion: core.apoxy.dev/v1alpha
 kind: TunnelNode
 metadata:
@@ -6,7 +23,7 @@ spec:
   egressGateway:
     enabled: true
 ---
-# Add just your backend for aibrix
+# Backend for vLLM deployments
 apiVersion: core.apoxy.dev/v1alpha
 kind: Backend
 metadata:
@@ -15,7 +32,7 @@ spec:
   endpoints:
     - fqdn: envoy-aibrix-system-aibrix-eg-903790dc.envoy-gateway-system.UNIQUE-TEMPNAME.tun.apoxy.net
 ---
-# Add just your route for aibrix
+# HTTPRoute for vLLM deployments
 apiVersion: gateway.apoxy.dev/v1
 kind: HTTPRoute
 metadata:
@@ -29,6 +46,53 @@ spec:
     - 'TEMPNAME.trainy.us'
   rules:
     - backendRefs:
-        - kind: Backend
-          name: UNIQUE-TEMPNAME-backend
-          port: 80
+      - kind: Backend
+        name: UNIQUE-TEMPNAME-backend
+        port: 80
+      timeouts:
+        request: "60s"
+---
+# Backend for general deployments
+apiVersion: core.apoxy.dev/v1alpha
+kind: Backend
+metadata:
+  name: UNIQUE-TEMPNAME-backend2
+spec:
+  endpoints:
+    - fqdn: ingress-nginx-controller.keda.UNIQUE-TEMPNAME.tun.apoxy.net
+---
+# HTTPRoute for general deployments
+apiVersion: gateway.apoxy.dev/v1
+kind: HTTPRoute
+metadata:
+  name: UNIQUE-TEMPNAME-route2
+spec:
+  parentRefs:
+    - name: default 
+      kind: Gateway
+      port: 443
+  hostnames:
+    - 'TEMPNAME2.trainy.us'
+  rules:
+    - backendRefs:
+      - kind: Backend
+        name: UNIQUE-TEMPNAME-backend2
+        port: 80
+      timeouts:
+        request: "60s"
+
+# KEDA proxy service (1 per cluster) (For general deployments)
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: keda-proxy
+  namespace: default
+spec:
+  type: ExternalName
+  externalName: keda-add-ons-http-interceptor-proxy.keda
+  ports:
+    - name: http
+      port: 8080
+      protocol: TCP
+      targetPort: 8080

konduktor/serving.py

@@ -49,11 +49,15 @@ class Serving:
         if min_replicas is None:
             min_replicas = max_replicas
         if max_replicas is None:
-            max_replicas = min_replicas
+            # Edge case: if min_replicas is 0, set max_replicas to 1
+            if min_replicas == 0:
+                max_replicas = 1
+            else:
+                max_replicas = min_replicas
 
-        if min_replicas is not None and min_replicas <= 0:
+        if min_replicas is not None and min_replicas < 0:
             with ux_utils.print_exception_no_traceback():
-                raise ValueError('min_replicas must be >= 1')
+                raise ValueError('min_replicas must be >= 0')
 
         if (
             max_replicas is not None
@@ -139,9 +143,9 @@ class Serving:
 
     def to_yaml_config(self) -> Dict[str, Union[int, str]]:
         config: Dict[str, Union[int, str]] = {
-            'min_replicas': self._min_replicas or 1,
-            'max_replicas': self._max_replicas or 1,
-            'ports': self._ports or 8000,
+            'min_replicas': self._min_replicas if self._min_replicas is not None else 1,
+            'max_replicas': self._max_replicas if self._max_replicas is not None else 1,
+            'ports': self._ports if self._ports is not None else 8000,
         }
         # Only include probe if it's not None
         if self._probe is not None:

konduktor/task.py

@@ -567,6 +567,13 @@ class Task:
                     f'less than min_replicas ({serving.min_replicas})'
                 )
 
+        if serving.max_replicas == 0 and serving.min_replicas == 0:
+            with ux_utils.print_exception_no_traceback():
+                raise ValueError(
+                    f'max_replicas ({serving.max_replicas}) and '
+                    f'min_replicas ({serving.min_replicas}) cannot both be 0'
+                )
+
         if isinstance(serving, konduktor.Serving):
             serving = serving
         self.serving = serving