kubetorch 0.2.0__py3-none-any.whl

kubetorch/serving/templates/kt_setup_template.sh.j2

@@ -0,0 +1,81 @@
+# Increase file descriptor limit for large-scale distributed jobs
+ulimit -n 65536
+
+{% if python_path %}
+export PATH="{{ python_path }}:$PATH"
+if command -v "{{ python_path }}" &> /dev/null; then
+  python_bin="{{ python_path }}"
+fi
+{% endif %}
+# If the user set the python_path to exact executable, then we'll use it directly here but adding it to PATH
+# above will have little effect. If they set it to a directory, then this command check will fail as desired,
+# and we'll then look for python3 or python in PATH (starting with their directory) as desired.
+if [[ -z "$python_bin" ]]; then
+  if command -v python3 &> /dev/null; then
+    python_bin="python3"
+  elif command -v python &> /dev/null; then
+    python_bin="python"
+  else
+    echo "Error: Neither python3 nor python found in PATH. Please set python_path to a valid Python executable."
+    exit 1
+  fi
+fi
+echo "Using Python binary: $python_bin"
+
+{% if not freeze %}
+if ! command -v rsync &> /dev/null; then
+  apt-get update && apt-get install -y rsync
+fi
+if ! command -v nohup &> /dev/null; then
+  apt-get update && apt-get install -y coreutils
+fi
+
+{% if install_cmd %}
+# Use the explicitly provided install command
+uv_pip_cmd="{{ install_cmd }}"
+{% else %}
+
+if $python_bin -c "import sys; exit(0 if sys.prefix != sys.base_prefix else 1)" 2>/dev/null; then
+  install_flags=""
+else
+  install_flags="--system --break-system-packages"
+fi
+
+# Check if uv is available and set the appropriate command
+if command -v uv &> /dev/null; then
+  # Use system-wide uv with the detected Python interpreter
+  uv_pip_cmd="uv pip install $install_flags --python=$python_bin"
+elif $python_bin -m uv --version &> /dev/null; then
+  # Use Python module uv - it inherently uses the right Python
+  uv_pip_cmd="$python_bin -m uv pip install $install_flags"
+else
+  # Install uv as a Python module and use it
+  echo "uv not found, installing it..."
+  $python_bin -m pip install uv
+  uv_pip_cmd="$python_bin -m uv pip install $install_flags"
+fi
+{% endif %}
+
+# Export the install command as an environment variable for use in applications
+echo "Setting KT_PIP_INSTALL_CMD env var to $uv_pip_cmd"
+export KT_PIP_INSTALL_CMD="$uv_pip_cmd"
+echo "$uv_pip_cmd" > .kt/kt_pip_install_cmd
+
+{% if install_url %}
+  $uv_pip_cmd "kubetorch[server] @ {{ install_url }}"
+  {% if install_otel %}
+    $uv_pip_cmd "kubetorch[otel] @ {{ install_url }}"
+  {% endif %}
+{% else %}
+  {{ rsync_kt_editable_cmd }}
+  $uv_pip_cmd -e "python_client[server]"
+  {% if install_otel %}
+    $uv_pip_cmd -e "python_client[otel]"
+  {% endif %}
+{% endif %}
+
+{% endif %}
+
+$python_bin -m uvicorn kubetorch.servers.http.http_server:app \
+    --host 0.0.0.0 \
+    --port {{ server_port }}

kubetorch/serving/templates/pod_template.yaml

@@ -0,0 +1,194 @@
+{% if service_account_name is not none %}
+serviceAccountName: {{ service_account_name }}
+{% endif %}
+
+{% if priority_class_name is not none %}
+priorityClassName: {{ priority_class_name }}
+{% endif %}
+
+{% if queue_name is not none %}
+schedulerName: {{ scheduler_name }}
+{% endif %}
+
+{% if gpu_anti_affinity is sameas true %}
+affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: nvidia.com/gpu
+              operator: DoesNotExist
+            - key: eks.amazonaws.com/instance-gpu-count
+              operator: DoesNotExist
+            - key: cloud.google.com/gke-accelerator
+              operator: DoesNotExist
+{% endif %}
+
+{% if node_selector is not none %}
+nodeSelector:
+  {% for key, value in node_selector.items() %}
+  {{ key }}: {{ value }}
+  {% endfor %}
+{% endif %}
+
+{% if tolerations is not none and tolerations|length > 0 %}
+tolerations:
+{% for tol in tolerations %}
+  - key: "{{ tol.key }}"
+    operator: "{{ tol.operator }}"
+    value: "{{ tol.value }}"
+    effect: "{{ tol.effect }}"
+{% endfor %}
+{% endif %}
+
+timeoutSeconds: {{ launch_timeout }}
+containers:
+  - name: kubetorch
+    image: {{ server_image }}
+    {% if image_pull_policy is not none %}
+    imagePullPolicy: {{ image_pull_policy }}
+    {% endif %}
+    {% if working_dir is not none %}
+    workingDir: {{ working_dir }}
+    {% endif %}
+    ports:
+      - name: http1
+        containerPort: {{ server_port }}
+    command: ["/bin/bash", "-c"]
+    {% if not freeze %}
+    securityContext:
+      capabilities:
+        add:
+          - "SYS_PTRACE"
+    {% endif %}
+    args:
+      - |
+{{ setup_script | indent(8, true) }}
+
+    env:
+      # Pod metadata available via the Kubernetes Downward API
+      - name: POD_NAME
+        valueFrom:
+          fieldRef:
+            fieldPath: metadata.name
+      - name: POD_NAMESPACE
+        valueFrom:
+          fieldRef:
+            fieldPath: metadata.namespace
+      - name: POD_IP
+        valueFrom:
+          fieldRef:
+            fieldPath: status.podIP
+      - name: POD_UUID
+        valueFrom:
+          fieldRef:
+            fieldPath: metadata.uid
+      - name: MODULE_NAME
+        valueFrom:
+          fieldRef:
+            fieldPath: metadata.labels['kubetorch.com/module']
+      - name: KUBETORCH_VERSION
+        valueFrom:
+          fieldRef:
+            fieldPath: metadata.labels['kubetorch.com/version']
+      - name: UV_LINK_MODE
+        value: "copy"  # Suppress the hardlink warning
+      - name: OTEL_EXPORTER_OTLP_ENDPOINT
+        value: "kubetorch-otel-opentelemetry-collector.kubetorch-monitoring.svc.cluster.local:4317"
+      - name: OTEL_EXPORTER_OTLP_PROTOCOL
+        value: "grpc"
+      - name: OTEL_TRACES_EXPORTER
+        value: "otlp"
+      - name: OTEL_PROPAGATORS
+        value: "tracecontext,baggage"
+      - name: KT_OTEL_ENABLED
+        value: "{{ otel_enabled }}"
+      - name: KT_SERVER_PORT
+        value: "{{ server_port }}"
+      - name: KT_FREEZE
+        value: "{{ freeze }}"
+      {% if inactivity_ttl is not none %}
+      - name: KT_INACTIVITY_TTL
+        value: "{{ inactivity_ttl }}"
+      {% endif %}
+      {% for key, value in config_env_vars.items() %}
+      - name: {{ key }}
+        value: "{{ value }}"
+      {% endfor %}
+      {% if env_vars is not none and env_vars|length > 0 %}
+      {% for key, value in env_vars.items() %}
+      - name: {{ key }}
+        value: "{{ value }}"
+      {% endfor %}
+      {% endif %}
+      {% for secret in secret_env_vars %}
+      {% for key in secret.env_vars %}
+      - name: {{ key }}
+        valueFrom:
+          secretKeyRef:
+            name: {{ secret.secret_name }}
+            key: {{ key }}
+      {% endfor %}
+      {% endfor %}
+    volumeMounts:
+      - mountPath: /dev/shm
+        name: dshm
+      {% for secret in secret_volumes %}
+      - name: {{ secret.name }}
+        mountPath: {{ secret.path }}
+        readOnly: true
+      {% endfor %}
+      {% if volume_mounts is not none and volume_mounts|length > 0 %}
+      {% for mount in volume_mounts %}
+      - name: {{ mount.name }}
+        mountPath: {{ mount.mountPath }}
+      {% endfor %}
+      {% endif %}
+    resources:
+      {{ resources | tojson }}
+    # TODO: do we want these health checks?
+    # Note: Knative won't consider the service ready to receive traffic until the probe succeeds at least once
+    # Initial readiness check
+    startupProbe:
+      httpGet:
+        path: /health
+        port: {{ server_port }}
+      initialDelaySeconds: 0
+      periodSeconds: 5
+      timeoutSeconds: 2
+      failureThreshold: {{ launch_timeout // 5 }}
+    readinessProbe:
+      httpGet:
+        path: /health
+        port: {{ server_port }}
+      periodSeconds: 3
+      successThreshold: 1
+      failureThreshold: 5
+    # Ongoing health monitoring with less frequent checks
+    livenessProbe:
+      httpGet:
+        path: /health
+        port: {{ server_port }}
+      periodSeconds: 30
+      timeoutSeconds: 1
+      failureThreshold: 3
+
+volumes:
+  - name: dshm
+    emptyDir:
+      medium: Memory
+      {% if shm_size_limit is not none %}
+      sizeLimit: {{ shm_size_limit }}
+      {% endif %}
+  {% for secret in secret_volumes %}
+  - name: {{ secret.name }}
+    secret:
+      secretName: {{ secret.secret_name }}
+  {% endfor %}
+  {% if volume_specs is not none and volume_specs|length > 0 %}
+  {% for spec in volume_specs %}
+  - name: {{ spec.name }}
+    persistentVolumeClaim:
+      claimName: {{ spec.persistentVolumeClaim.claimName }}
+  {% endfor %}
+  {% endif %}

kubetorch/serving/templates/raycluster_service_template.yaml

@@ -0,0 +1,42 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ name }}
+  namespace: {{ namespace }}
+  annotations: {{ annotations | tojson }}
+  labels: {{ labels | tojson }}
+spec:
+  {% if distributed %}
+  clusterIP: None  # Headless service for Ray pod discovery
+  {% else %}
+  sessionAffinity: ClientIP  # Ensure requests from same client go to same pod
+  {% endif %}
+  selector:
+    kubetorch.com/service: {{ deployment_name }}
+    kubetorch.com/module: {{ module_name }}
+    ray.io/node-type: head  # Only select head node pods
+  ports:
+    - name: http
+      port: 80
+      targetPort: {{ server_port }}
+      protocol: TCP
+    - name: ray-gcs
+      port: 6379
+      targetPort: 6379
+      protocol: TCP
+    - name: ray-object-mgr
+      port: 8076
+      targetPort: 8076
+      protocol: TCP
+    - name: ray-node-mgr
+      port: 8077
+      targetPort: 8077
+      protocol: TCP
+    - name: ray-dashboard
+      port: 8265
+      targetPort: 8265
+      protocol: TCP
+    - name: ray-metrics
+      port: 8080
+      targetPort: 8080
+      protocol: TCP

kubetorch/serving/templates/raycluster_template.yaml

@@ -0,0 +1,35 @@
+apiVersion: ray.io/v1
+kind: RayCluster
+metadata:
+  name: {{ name }}
+  namespace: {{ namespace }}
+  annotations: {{ annotations | tojson }}
+  labels: {{ labels | tojson }}
+spec:
+  rayVersion: "2.8.0"
+  enableInTreeAutoscaling: false
+  headGroupSpec:
+    rayStartParams:
+      dashboard-host: "0.0.0.0"
+      port: "6379"
+      object-manager-port: "8076"
+      node-manager-port: "8077"
+      dashboard-port: "8265"
+      metrics-export-port: "8080"
+    replicas: 1
+    template:
+      metadata:
+        annotations: {{ template_annotations | tojson }}
+        labels: {{ head_template_labels | tojson }}
+      spec: {{ pod_template | tojson }}
+  workerGroupSpecs:
+  - groupName: worker-group
+    rayStartParams: {}
+    minReplicas: 0
+    maxReplicas: {{ worker_replicas }}
+    replicas: {{ worker_replicas }}
+    template:
+      metadata:
+        annotations: {{ template_annotations | tojson }}
+        labels: {{ worker_template_labels | tojson }}
+      spec: {{ pod_template | tojson }}

kubetorch/serving/templates/service_template.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ name }}
+  namespace: {{ namespace }}
+  annotations: {{ annotations | tojson }}
+  labels: {{ labels | tojson }}
+spec:
+  {% if distributed %}
+  clusterIP: None  # Headless service for distributed pod discovery
+  {% else %}
+  sessionAffinity: ClientIP  # Ensure requests from same client go to same pod
+  {% endif %}
+  selector:
+    kubetorch.com/service: {{ deployment_name }}
+    kubetorch.com/module: {{ module_name }}  # Only deployment pods have this set, so allows us to exclude the jump pod
+  ports:
+    - name: http
+      port: 80
+      targetPort: {{ server_port }}
+      protocol: TCP

kubetorch/serving/templates/workerset_template.yaml

@@ -0,0 +1,36 @@
+# --- Headless Service selecting all pods with 'app=my-app-name' ---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ workerset_name }}
+  namespace: {{ namespace }}
+  annotations: {{ annotations | tojson }}
+  labels: {{ labels | tojson }}
+spec:
+  clusterIP: None # Make it headless
+  selector:
+    app: {{ workerset_name_app }} # Selects pods from any associated StatefulSets
+
+---
+# --- StatefulSet ---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ worker_group_name }}
+  namespace: {{ namespace }}
+  ownerReferences:
+    - apiVersion: v1
+      kind: Pod
+      name: {{ service_pod_name }}
+      uid: {{ service_pod_uid }}
+spec:
+  serviceName: {{ workerset_name }} # Must match the headless service name
+  replicas: {{ replicas }}
+  selector:
+    matchLabels:
+      app: {{ workerset_name_app }} # Matches the service selector
+  template:
+    metadata:
+      labels:
+        app: {{ workerset_name_app }} # Label for the service
+    spec: {{ pod_template | tojson }}