konduktor-nightly 0.1.0.dev20250915104603__py3-none-any.whl → 0.1.0.dev20251107104752__py3-none-any.whl

konduktor/manifests/aibrix-setup.yaml

@@ -0,0 +1,430 @@
+# Aibrix Setup - vLLM Deployment Infrastructure
+# 
+# This file sets up the infrastructure needed for vLLM (Aibrix) deployments:
+# 1. Envoy Gateway configuration for HTTP routing
+# 2. Aibrix Activator service for request-based autoscaling (KPA)
+# 3. HTTP route mirroring for prewarming vLLM models
+# 4. Lua script for extracting model names from OpenAI-compatible requests
+#
+# The activator tracks incoming requests and provides metrics to scale
+# vLLM deployments based on demand (requests per second).
+
+# This file is kept separate from apoxy setup files because it is 
+# only used in actual clusters, not in the test kind clusters.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: envoy-gateway-config
+  namespace: envoy-gateway-system
+data:
+  envoy-gateway.yaml: |
+    apiVersion: gateway.envoyproxy.io/v1alpha1
+    kind: EnvoyGateway
+    provider:
+      type: Kubernetes
+    gateway:
+      controllerName: gateway.envoyproxy.io/gatewayclass-controller
+    extensionApis:
+      enableEnvoyPatchPolicy: true
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: aibrix-activator
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: aibrix-activator
+  namespace: aibrix-activator
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: aibrix-activator
+rules:
+- apiGroups: ["apps"]
+  resources: ["deployments"]
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: aibrix-activator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: aibrix-activator
+subjects:
+- kind: ServiceAccount
+  name: aibrix-activator
+  namespace: aibrix-activator
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: activator-code
+  namespace: aibrix-activator
+data:
+  activator.py: |
+    import os, time, json
+    from collections import defaultdict, deque
+    from fastapi import FastAPI, Request
+    from fastapi.responses import PlainTextResponse, JSONResponse
+    import asyncio
+    from kubernetes import client, config
+
+    NAMESPACE = os.getenv("NAMESPACE", "default")
+    WINDOW_SEC = int(os.getenv("WINDOW_SEC", "30"))        # demand lookback
+    CAPACITY_RPS = float(os.getenv("CAPACITY_RPS", "1.0")) # per-replica capacity
+    MIN_WAKE = int(os.getenv("MIN_REPLICA_ON_WAKE", "1"))
+    MAX_REPLICAS = int(os.getenv("MAX_REPLICAS", "8"))
+    CLEANUP_INTERVAL = int(os.getenv("CLEANUP_INTERVAL", "300"))  # 5 minutes
+
+    app = FastAPI()
+    events = defaultdict(deque)  # key=(ns,model) -> deque[timestamps]
+
+    # Initialize Kubernetes client
+    try:
+        config.load_incluster_config()
+        k8s_apps_v1 = client.AppsV1Api()
+    except:
+        k8s_apps_v1 = None
+
+    def _prune(q, now):
+        while q and now - q[0] > WINDOW_SEC: q.popleft()
+
+    def _bump(ns, model):
+        now = time.time()
+        q = events[(ns, model)]
+        q.append(now)
+        _prune(q, now)
+
+    def _desired(ns, model):
+        now = time.time()
+        q = events[(ns, model)]
+        _prune(q, now)
+        rps = len(q) / max(WINDOW_SEC, 1)
+        if len(q) == 0: return 0
+        # Convert demand to desired replicas
+        import math
+        d = max(MIN_WAKE, math.ceil(rps / max(CAPACITY_RPS, 1e-6)))
+        return max(0, min(d, MAX_REPLICAS))
+
+    def _extract_model(headers, body_bytes):
+        # Prefer header (OpenAI-compatible)
+        m = headers.get("model") or headers.get("x-model")
+        if m: return m
+        # Try JSON body
+        try:
+            j = json.loads(body_bytes or b"{}")
+            if isinstance(j, dict):
+                # OpenAI schema: {"model": "...", ...}
+                if "model" in j and isinstance(j["model"], str):
+                    return j["model"]
+        except Exception:
+            pass
+        return None
+
+    def _get_existing_deployments():
+        """Get list of existing Aibrix deployments from Kubernetes"""
+        if not k8s_apps_v1:
+            return set()
+        try:
+            deployments = k8s_apps_v1.list_namespaced_deployment(
+                namespace=NAMESPACE,
+                label_selector="model.aibrix.ai/name"
+            )
+            return {d.metadata.name for d in deployments.items}
+        except Exception:
+            return set()
+
+    def _cleanup_stale_entries():
+        """Remove entries for deployments that no longer exist"""
+        if not k8s_apps_v1:
+            return
+        try:
+            existing_deployments = _get_existing_deployments()
+            # Remove entries for deployments that no longer exist
+            keys_to_remove = []
+            for (ns, model) in list(events.keys()):
+                if ns == NAMESPACE and model not in existing_deployments:
+                    keys_to_remove.append((ns, model))
+            
+            for key in keys_to_remove:
+                del events[key]
+                print(f"Cleaned up stale entry for deployment: {key[1]}")
+        except Exception as e:
+            print(f"Error during cleanup: {e}")
+
+    async def _cleanup_task():
+        """Background task to periodically clean up stale entries"""
+        while True:
+            await asyncio.sleep(CLEANUP_INTERVAL)
+            _cleanup_stale_entries()
+
+    @app.on_event("startup")
+    async def startup_event():
+        """Start background cleanup task"""
+        asyncio.create_task(_cleanup_task())
+
+    # Mirror endpoints (same as your API paths); quick 204 response
+    @app.post("/v1/completions")
+    @app.post("/v1/chat/completions")
+    async def mirrored(request: Request):
+        body = await request.body()
+        model = _extract_model(request.headers, body)
+        if model:
+            _bump(NAMESPACE, model)
+        return JSONResponse({"ok": True}, status_code=204)
+
+    # Catch-all POST (safety net if your gateway uses different paths)
+    @app.post("/{full_path:path}")
+    async def mirrored_generic(request: Request, full_path: str):
+        body = await request.body()
+        model = _extract_model(request.headers, body)
+        if model:
+            _bump(NAMESPACE, model)
+        return JSONResponse({"ok": True}, status_code=204)
+
+    # Prometheus-friendly aggregate endpoint: export ALL (ns, model)
+    @app.get("/metrics", response_class=PlainTextResponse)
+    async def metrics_all():
+        lines = []
+        # Idiomatic names
+        lines.append("# HELP vllm_deployment_replicas Number of suggested replicas.")
+        lines.append("# TYPE vllm_deployment_replicas gauge")
+        lines.append("# HELP vllm_observed_rps Incoming requests per second.")
+        lines.append("# TYPE vllm_observed_rps gauge")
+        now = time.time()
+        for (ns, model), q in list(events.items()):
+            _prune(q, now)
+            rps = len(q) / max(WINDOW_SEC, 1)
+            d = _desired(ns, model)
+            lines.append(f'vllm_deployment_replicas{{namespace="{ns}",model_name="{model}"}} {d}')
+            lines.append(f'vllm_observed_rps{{namespace="{ns}",model_name="{model}"}} {rps:.6f}')
+        # (Optional) keep legacy names with colons for back-compat
+        lines.append("# HELP vllm:deployment_replicas Number of suggested replicas.")
+        lines.append("# TYPE vllm:deployment_replicas gauge")
+        lines.append("# HELP vllm:observed_rps Incoming requests per second.")
+        lines.append("# TYPE vllm:observed_rps gauge")
+        now = time.time()
+        for (ns, model), q in list(events.items()):
+            _prune(q, now)
+            rps = len(q) / max(WINDOW_SEC, 1)
+            d = _desired(ns, model)
+            lines.append(f'vllm:deployment_replicas{{namespace="{ns}",model_name="{model}"}} {d}')
+            lines.append(f'vllm:observed_rps{{namespace="{ns}",model_name="{model}"}} {rps:.6f}')
+        return "\n".join(lines) + "\n"
+
+
+    # Metrics for KPA and Debugging
+    @app.get("/metrics/{ns}/{model}", response_class=PlainTextResponse)
+    async def metrics(ns: str, model: str):
+        d = _desired(ns, model)
+        now = time.time()
+        q = events[(ns, model)]
+        _prune(q, now)
+        rps = len(q) / max(WINDOW_SEC, 1)
+        return (
+            "# HELP vllm:deployment_replicas Number of suggested replicas.\n"
+            "# TYPE vllm:deployment_replicas gauge\n"
+            f'vllm:deployment_replicas{{namespace="{ns}",model_name="{model}"}} {d}\n'
+            "# HELP vllm:observed_rps Incoming requests per second.\n"
+            "# TYPE vllm:observed_rps gauge\n"
+            f'vllm:observed_rps{{namespace="{ns}",model_name="{model}"}} {rps:.2f}\n'
+        )
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: aibrix-activator
+  namespace: aibrix-activator
+spec:
+  replicas: 1
+  selector: { matchLabels: { app: aibrix-activator } }
+  template:
+    metadata: { labels: { app: aibrix-activator } }
+    spec:
+      containers:
+      - name: activator
+        image: python:3.11-slim
+        command: ["bash","-lc"]
+        args:
+          - |
+            pip install fastapi uvicorn kubernetes >/dev/null && \
+            uvicorn activator:app --host 0.0.0.0 --port 8080
+        env:
+        - { name: NAMESPACE, value: "default" }
+        - { name: WINDOW_SEC, value: "30" }
+        - { name: CAPACITY_RPS, value: "1.0" }
+        - { name: MIN_REPLICA_ON_WAKE, value: "1" }
+        - { name: MAX_REPLICAS, value: "8" }
+        - { name: CLEANUP_INTERVAL, value: "300" }
+        ports: [{containerPort: 8080}]
+        volumeMounts:
+        - { name: code, mountPath: /app/activator.py, subPath: activator.py }
+        workingDir: /app
+      serviceAccountName: aibrix-activator
+      volumes:
+      - name: code
+        configMap: { name: activator-code }
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: aibrix-activator
+  namespace: aibrix-activator
+  annotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/port: "8080"
+    prometheus.io/path: "/metrics"
+  labels:
+    app: aibrix-activator
+    prometheus-discovery: "true"
+spec:
+  selector: { app: aibrix-activator }
+  ports:
+  - name: http
+    port: 8080
+    targetPort: 8080
+    protocol: TCP
+  type: ClusterIP
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: aibrix-activator
+  namespace: prometheus
+  labels:
+    app: aibrix-activator
+spec:
+  selector:
+    matchLabels:
+      app: aibrix-activator
+  namespaceSelector:
+    matchNames:
+    - aibrix-activator
+  endpoints:
+  - port: http
+    path: /metrics
+---
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: vllm-deployments
+  namespace: prometheus
+  labels:
+    app: vllm-deployments
+spec:
+  selector:
+    matchLabels:
+      prometheus-discovery: "true"
+  namespaceSelector:
+    matchNames:
+    - default
+  endpoints:
+  - port: serve
+    path: /metrics
+---
+apiVersion: gateway.networking.k8s.io/v1beta1
+kind: ReferenceGrant
+metadata:
+  name: allow-httproute-to-activator
+  namespace: aibrix-activator
+spec:
+  from:
+  - group: gateway.networking.k8s.io
+    kind: HTTPRoute
+    namespace: aibrix-system
+  to:
+  - group: ""
+    kind: Service
+    name: aibrix-activator
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: activator-mirror-sink
+  namespace: aibrix-system
+spec:
+  parentRefs:
+  - group: gateway.networking.k8s.io
+    kind: Gateway
+    name: aibrix-eg
+    namespace: aibrix-system
+  rules:
+  - matches:
+    - path:
+        type: PathPrefix
+        value: /__activator_sink__
+    backendRefs:
+    - name: aibrix-activator
+      namespace: aibrix-activator
+      port: 8080
+---
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: EnvoyPatchPolicy
+metadata:
+  name: prewarm-completions-lua
+  namespace: aibrix-system
+spec:
+  targetRef:
+    group: gateway.networking.k8s.io
+    kind: Gateway
+    name: aibrix-eg
+  type: JSONPatch
+  jsonPatches:
+    - type: "type.googleapis.com/envoy.config.listener.v3.Listener"
+      name: "aibrix-system/aibrix-eg/http"
+      operation:
+        op: add
+        path: "/default_filter_chain/filters/0/typed_config/http_filters/0"
+        value:
+          name: envoy.filters.http.lua
+          typed_config:
+            "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua
+            inlineCode: |
+              function envoy_on_request(handle)
+                local path = handle:headers():get(":path") or ""
+                if string.find(path, "^/v1/completions") or string.find(path, "^/v1/chat/completions") then
+                  -- Try to get model from header first
+                  local model = handle:headers():get("model") or ""
+                  
+                  -- If no model in header, try to extract from JSON body
+                  if model == "" then
+                    local ct = handle:headers():get("content-type") or ""
+                    if string.find(ct:lower(), "application/json") then
+                      local body = handle:body()
+                      if body and body:length() > 0 then
+                        local raw = body:getBytes(0, math.min(body:length(), 1024))
+                        -- Simple regex to extract model from JSON: "model":"value"
+                        local model_match = raw:match('"model"%s*:%s*"([^"]+)"')
+                        if model_match then
+                          model = model_match
+                        end
+                      end
+                    end
+                  end
+                  
+                  -- Only proceed if we have a model
+                  if model ~= "" then
+                    -- fire-and-forget wake signal; very short timeout
+                    pcall(function()
+                      handle:httpCall(
+                        "httproute/aibrix-system/activator-mirror-sink/rule/0",
+                        {
+                          [":method"] = "POST",
+                          [":path"] = "/v1/completions",
+                          [":authority"] = "aibrix-activator.aibrix-activator.svc.cluster.local",
+                          ["content-type"] = "application/json",
+                          ["model"] = model
+                        },
+                        "{}",
+                        5 -- ms
+                      )
+                    end)
+                  end
+                end
+              end

konduktor/manifests/apoxy-setup.yaml

@@ -1,8 +1,25 @@
+# Apoxy Setup (Part 1/3) - Core Infrastructure
+#
+# This file sets up the core Apoxy infrastructure for external access to deployments:
+# 1. Apoxy system namespace and RBAC
+# 2. Kubeconfig secret for cluster access (populated by CI)
+# 3. Apoxy tunnel controller and proxy services
+# 4. Network policies for cross-namespace access
+#
+# Split into 2 files because:
+# - apoxy-setup.yaml: Core infrastructure (1 per cluster) (needs to be applied first)
+# - apoxy-setup2.yaml: All routing rules for both deployment types
+
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: apoxy-system
+---
 apiVersion: v1
 kind: Secret
 metadata:
   name: trainy-kubeconfig
-  namespace: default
+  namespace: apoxy-system
 type: Opaque
 data:
   # this gets replaced by buildkite CI secret APOXY_AUTH
@@ -13,7 +30,7 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: kube-controller
-  namespace: default
+  namespace: apoxy-system
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -35,13 +52,13 @@ roleRef:
 subjects:
 - kind: ServiceAccount
   name: kube-controller
-  namespace: default
+  namespace: apoxy-system
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: kube-controller
-  namespace: default
+  namespace: apoxy-system
   labels:
     app: kube-controller
 spec:
@@ -85,7 +102,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: kube-controller
-  namespace: default
+  namespace: apoxy-system
   labels:
     app: kube-controller
 spec:
@@ -101,7 +118,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: apoxy-config
-  namespace: default
+  namespace: apoxy-system
 data:
   config.yaml: |
     apiVersion: config.apoxy.dev/v1alpha1
@@ -118,7 +135,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: apoxy
-  namespace: default
+  namespace: apoxy-system
   labels:
     app: apoxy
 spec:
@@ -133,7 +150,7 @@ spec:
     spec:
       containers:
       - name: apoxy
-        image: apoxy/apoxy:v0.11.14
+        image: apoxy/apoxy:v0.11.18
         command: ["apoxy", "tunnel", "run", "UNIQUE-TEMPNAME", "--insecure-skip-verify"]
         volumeMounts:
         - name: kubeconfig-volume
@@ -148,4 +165,20 @@ spec:
           secretName: trainy-kubeconfig
       - name: apoxy-config-volume
         configMap:
-          name: apoxy-config
+          name: apoxy-config
+---
+# NetworkPolicy to allow Apoxy to reach services in other namespaces
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: apoxy-cross-namespace-access
+  namespace: apoxy-system
+spec:
+  podSelector:
+    matchLabels:
+      app: apoxy
+  policyTypes:
+  - Egress
+  egress:
+  # Allow all egress traffic
+  - {}

konduktor/manifests/apoxy-setup2.yaml

@@ -1,3 +1,20 @@
+# Apoxy Setup (Part 2/2) - Deployment Routing
+#
+# This file sets up Apoxy routing for both vLLM and general deployments:
+# 1. TunnelNode for secure tunnel connection
+# 2. Backend for vLLM pointing to Envoy Gateway
+# 3. HTTPRoute for company.trainy.us -> vLLM deployments
+# 4. Backend for general deployments pointing to nginx ingress
+# 5. HTTPRoute for company2.trainy.us -> general deployments
+# 6. KEDA proxy service for HTTP autoscaling
+# 7. 60s timeout for all requests
+#
+# Split into 2 files because:
+# - apoxy-setup.yaml: Core infrastructure (1 per cluster) (needs to be applied first)
+# - apoxy-setup2.yaml: All routing rules for both deployment types
+
+# NOTE: TunnelNode should technically be in the first apoxy-setup.yaml but it
+# needs to be created after the core infrastructure is created, so we put it here.
 apiVersion: core.apoxy.dev/v1alpha
 kind: TunnelNode
 metadata:
@@ -6,7 +23,7 @@ spec:
   egressGateway:
     enabled: true
 ---
-# Add just your backend for aibrix
+# Backend for vLLM deployments
 apiVersion: core.apoxy.dev/v1alpha
 kind: Backend
 metadata:
@@ -15,7 +32,7 @@ spec:
   endpoints:
     - fqdn: envoy-aibrix-system-aibrix-eg-903790dc.envoy-gateway-system.UNIQUE-TEMPNAME.tun.apoxy.net
 ---
-# Add just your route for aibrix
+# HTTPRoute for vLLM deployments
 apiVersion: gateway.apoxy.dev/v1
 kind: HTTPRoute
 metadata:
@@ -29,6 +46,53 @@ spec:
     - 'TEMPNAME.trainy.us'
   rules:
     - backendRefs:
-        - kind: Backend
-          name: UNIQUE-TEMPNAME-backend
-          port: 80
+      - kind: Backend
+        name: UNIQUE-TEMPNAME-backend
+        port: 80
+      timeouts:
+        request: "60s"
+---
+# Backend for general deployments
+apiVersion: core.apoxy.dev/v1alpha
+kind: Backend
+metadata:
+  name: UNIQUE-TEMPNAME-backend2
+spec:
+  endpoints:
+    - fqdn: keda-ingress-nginx-controller.keda.UNIQUE-TEMPNAME.tun.apoxy.net
+---
+# HTTPRoute for general deployments
+apiVersion: gateway.apoxy.dev/v1
+kind: HTTPRoute
+metadata:
+  name: UNIQUE-TEMPNAME-route2
+spec:
+  parentRefs:
+    - name: default 
+      kind: Gateway
+      port: 443
+  hostnames:
+    - 'TEMPNAME2.trainy.us'
+  rules:
+    - backendRefs:
+      - kind: Backend
+        name: UNIQUE-TEMPNAME-backend2
+        port: 80
+      timeouts:
+        request: "60s"
+
+# KEDA proxy service (1 per cluster) (For general deployments)
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: keda-proxy
+  namespace: default
+spec:
+  type: ExternalName
+  externalName: keda-add-ons-http-interceptor-proxy.keda
+  ports:
+    - name: http
+      port: 8080
+      protocol: TCP
+      targetPort: 8080

konduktor/resource.py

@@ -59,6 +59,8 @@ class Resources:
         # Internal use only.
         # pylint: disable=invalid-name
         _cluster_config_overrides: Optional[Dict[str, Any]] = None,
+        # used to prevent double validation of image (would happen from overrides)
+        _validate_image: bool = True,
     ):
         """Initialize a Resources object.
 
@@ -124,7 +126,8 @@ class Resources:
         if isinstance(image_id, str):
             self._image_id = image_id.strip()
             # Validate Docker image format and existence
-            validator.validate_and_warn_image(self._image_id, 'task')
+            if _validate_image:
+                validator.validate_and_warn_image(self._image_id, 'task')
 
         self._labels = labels
         self._cluster_config_overrides = _cluster_config_overrides
@@ -435,15 +438,19 @@ class Resources:
 
     def copy(self, **override) -> 'Resources':
         """Returns a copy of the given Resources."""
+        # used to prevent double validation of image (would happen from overrides)
+        new_image_id = override.pop('image_id', self.image_id)
         resources = Resources(
             cloud=override.pop('cloud', self.cloud),
             cpus=override.pop('cpus', self._cpus),
             memory=override.pop('memory', self.memory),
             accelerators=override.pop('accelerators', self.accelerators),
             disk_size=override.pop('disk_size', self.disk_size),
-            image_id=override.pop('image_id', self.image_id),
+            image_id=new_image_id,
             labels=override.pop('labels', self.labels),
             job_config=override.pop('job_config', self.job_config),
+            # used to prevent double validation of image (would happen from overrides)
+            _validate_image=(new_image_id != self.image_id),
         )
         assert len(override) == 0
         return resources

konduktor/serving.py

@@ -49,11 +49,15 @@ class Serving:
         if min_replicas is None:
             min_replicas = max_replicas
         if max_replicas is None:
-            max_replicas = min_replicas
+            # Edge case: if min_replicas is 0, set max_replicas to 1
+            if min_replicas == 0:
+                max_replicas = 1
+            else:
+                max_replicas = min_replicas
 
-        if min_replicas is not None and min_replicas <= 0:
+        if min_replicas is not None and min_replicas < 0:
             with ux_utils.print_exception_no_traceback():
-                raise ValueError('min_replicas must be >= 1')
+                raise ValueError('min_replicas must be >= 0')
 
         if (
             max_replicas is not None
@@ -139,9 +143,9 @@ class Serving:
 
     def to_yaml_config(self) -> Dict[str, Union[int, str]]:
         config: Dict[str, Union[int, str]] = {
-            'min_replicas': self._min_replicas or 1,
-            'max_replicas': self._max_replicas or 1,
-            'ports': self._ports or 8000,
+            'min_replicas': self._min_replicas if self._min_replicas is not None else 1,
+            'max_replicas': self._max_replicas if self._max_replicas is not None else 1,
+            'ports': self._ports if self._ports is not None else 8000,
         }
         # Only include probe if it's not None
         if self._probe is not None: