terradev-cli 5.3.8__tar.gz → 5.3.9__tar.gz

{terradev_cli-5.3.8 → terradev_cli-5.3.9}/Deployment Package/BNF_GRAMMAR.md

@@ -1,5 +1,5 @@
 # Terradev CLI BNF Grammar
-# Complete syntax specification for all commands and options — v5.3.8
+# Complete syntax specification for all commands and options — v5.3.9
 
 <terradev-cli> ::= <global-options> <command>
 
@@ -605,8 +605,8 @@
 <lora-commands> ::= lora <lora-subcommand>
 
 <lora-subcommand> ::=
-    | list         -e <url> [--api-key <string>]
-    | add          -e <url> -n <string> --path <path> [--api-key <string>]
+    | list         -e <url> [--api-key <string>] [--registry]
+    | add          -e <url> -n <string> --path <path> [--api-key <string>] [--register] [-b <string>] [--rank <integer>]
     | remove       -e <url> -n <string> [--api-key <string>]
     | register     -n <string> --path <path> -b <string> [--rank <integer>] [--tenant <string>] [--metadata <string>]
     | versions     -n <string>
@@ -615,6 +615,31 @@
     | rollback     -n <string> [-v <string>] [--replicas <string>]
     | drift-check  -n <string> [-v <string>] [-t <float>] [--source <string>]
     | cost-report  [-d <integer>] [-a <string>] [-t <string>]
+    | lorax        <lorax-subcommand>
+    | peft         <peft-subcommand>
+
+
+# ── LoRAX Subcommands (under lora) ──
+
+<lorax-subcommand> ::=
+    | deploy    -m <string> [--host <string>] [-p <integer>] [--quantization <lorax-quant>] [--gpu-memory-fraction <float>] [--max-loras <integer>] [--docker] [--k8s] [--namespace <string>]
+    | test      [--host <string>] [-p <integer>]
+    | list-adapters [--host <string>] [-p <integer>]
+    | load-adapter -a <string> [--adapter-name <string>] [--host <string>] [-p <integer>]
+    | unload-adapter -a <string> [--host <string>] [-p <integer>]
+    | sync-registry [--host <string>] [-p <integer>] [-a <string>]
+    | generate   -p <string> [-a <string>] [--max-tokens <integer>] [--temperature <float>] [--host <string>] [-p <integer>]
+
+<lorax-quant> ::= none | bitsandbytes | gptq | awq
+
+
+# ── PEFT Subcommands (under lora) ──
+
+<peft-subcommand> ::=
+    | import   -a <string> [--local-name <string>] [--token <string>] [--register] [-b <string>] [--rank <integer>]
+    | list
+    | validate -p <path>
+    | delete   -a <string>
 
 
 # ════════════════════════════════════════════════════════════════════

{terradev_cli-5.3.8 → terradev_cli-5.3.9}/Deployment Package/LIFECYCLES.md

@@ -1,4 +1,4 @@
-# Terradev CLI Lifecycles (v5.1.5)
+# Terradev CLI Lifecycles (v5.3.9)
 
 Complete end-to-end workflows covering nearly every command in the CLI. These have been updated to match the current CLI signatures and remove deprecated commands.
 
@@ -1388,6 +1388,158 @@ terradev local scan \
 terradev local scan --register --name workstation-4090
 ```
 
+---
+
+## Lifecycle 17 — LoRAX Multi-LoRA Inference (NEW)
+
+**Goal:** Deploy LoRAX server for serving thousands of fine-tuned models on a single GPU with dynamic adapter loading.
+
+### Phase 1: Import Adapter from HuggingFace
+
+```bash
+# Import LoRA adapter from HuggingFace
+terradev lora peft import \
+  -a vineetsharma/qlora-adapter-Mistral-7B-Instruct-v0.1-gsm8k \
+  --local-name gsm8k-adapter
+
+# Import and register in Terradev registry
+terradev lora peft import \
+  -a vineetsharma/qlora-adapter-Mistral-7B-Instruct-v0.1-gsm8k \
+  --local-name gsm8k-adapter \
+  --register \
+  --base-model mistralai/Mistral-7B-Instruct-v0.1
+
+# Import from private repo
+terradev lora peft import \
+  -a username/private-adapter \
+  --token hf_xxx
+```
+
+### Phase 2: List Local Adapters
+
+```bash
+# List all imported adapters
+terradev lora peft list
+
+# Validate adapter structure
+terradev lora peft validate -p ~/.terradev/peft_adapters/username--adapter
+```
+
+### Phase 3: Deploy LoRAX Server (Docker)
+
+```bash
+# Deploy LoRAX with Docker
+terradev lora lorax deploy \
+  -m mistralai/Mistral-7B-Instruct-v0.1 \
+  --docker
+
+# Deploy with quantization
+terradev lora lorax deploy \
+  -m mistralai/Mistral-7B-Instruct-v0.1 \
+  --quantization bitsandbytes \
+  --max-loras 16 \
+  --docker
+```
+
+### Phase 4: Deploy LoRAX Server (Kubernetes)
+
+```bash
+# Deploy LoRAX to Kubernetes
+terradev lora lorax deploy \
+  -m mistralai/Mistral-7B-Instruct-v0.1 \
+  --k8s \
+  --namespace lorax
+
+# Install via Helm
+helm install lorax ./clusters/lorax-template/helm \
+  -f clusters/lorax-template/helm/values-lorax.yaml \
+  --set model.id=mistralai/Mistral-7B-Instruct-v0.1 \
+  --set maxLoras=8
+```
+
+### Phase 5: Test LoRAX Server
+
+```bash
+# Test server connectivity
+terradev lora lorax test --host localhost --port 8080
+
+# Test with custom host/port
+terradev lora lorax test --host 10.0.0.1 --port 8080
+```
+
+### Phase 6: Load Adapters
+
+```bash
+# Load adapter from HuggingFace
+terradev lora lorax load-adapter \
+  -a vineetsharma/qlora-adapter-Mistral-7B-Instruct-v0.1-gsm8k
+
+# Load local adapter with custom name
+terradev lora lorax load-adapter \
+  -a ~/.terradev/peft_adapters/gsm8k-adapter \
+  --adapter-name gsm8k-adapter \
+  --host localhost --port 8080
+```
+
+### Phase 7: List Loaded Adapters
+
+```bash
+# List all loaded adapters
+terradev lora lorax list-adapters
+
+# List from specific server
+terradev lora lorax list-adapters --host 10.0.0.1 --port 8080
+```
+
+### Phase 8: Generate Text
+
+```bash
+# Generate with base model
+terradev lora lorax generate \
+  -p "What is 2+2?" \
+  --max-tokens 64
+
+# Generate with adapter
+terradev lora lorax generate \
+  -p "Natalia sold clips to 48 of her friends..." \
+  -a gsm8k-adapter \
+  --temperature 0.7
+```
+
+### Phase 9: Unload Adapter
+
+```bash
+# Unload adapter
+terradev lora lorax unload-adapter -a gsm8k-adapter
+```
+
+### Phase 10: Sync Registry
+
+```bash
+# Sync Terradev registry with LoRAX state
+terradev lora lorax sync-registry
+
+# Sync specific adapter
+terradev lora lorax sync-registry -a gsm8k-adapter
+```
+
+### Phase 11: Delete Local Adapter
+
+```bash
+# Delete imported adapter
+terradev lora peft delete -a username/adapter
+```
+
+### Phase 12: Cleanup
+
+```bash
+# Terminate LoRAX server
+docker stop <container-id>
+
+# Or delete Kubernetes deployment
+helm uninstall lorax -n lorax
+```
+
 ### Phase 4: View Hybrid Pool
 
 ```bash

{terradev_cli-5.3.8 → terradev_cli-5.3.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: terradev-cli
-Version: 5.3.8
+Version: 5.3.9
 Summary: An imperative command-line-interface for AI workload orchestration.
 Author-email: Terradev Team <team@terradev.com>
 Maintainer-email: Terradev Team <team@terradev.com>
@@ -111,6 +111,38 @@ Continued focus on lower cost, faster provisioning, and topology-aware execution
 
 Model agnostic. Dataset agnostic. GPU agnostic. Provider agnostic. The only thing Terradev is not agnostic about is correctness: it enforces topology, idempotency, and sequencing.
 
+**NOTES ON 5.3.9**
+
+Added **LoRAX (LoRA eXchange) integration** and **HuggingFace PEFT import** for production-grade multi-LoRA inference serving:
+
+- **LoRAX Service** (`ml_services/lorax_service.py`): Async HTTP client for Predibase LoRAX multi-LoRA inference server that serves thousands of fine-tuned models on a single GPU with dynamic adapter loading, heterogeneous continuous batching, and adapter exchange scheduling.
+
+- **LoRAX CLI commands** (`terradev lora lorax`):
+  ```bash
+  terradev lora lorax deploy -m mistralai/Mistral-7B-Instruct-v0.1 --docker
+  terradev lora lorax test --host localhost --port 8080
+  terradev lora lorax list-adapters
+  terradev lora lorax load-adapter -a vineetsharma/qlora-adapter-Mistral-7B-Instruct-v0.1-gsm8k
+  terradev lora lorax unload-adapter -a my-adapter
+  terradev lora lorax generate -p "What is 2+2?" -a my-adapter
+  terradev lora lorax sync-registry
+  ```
+
+- **PEFT Import Service** (`ml_services/peft_import_service.py`): Download, validate, and prepare LoRA adapters from HuggingFace using the PEFT library with auto-detection of rank, alpha, and target modules.
+
+- **PEFT CLI commands** (`terradev lora peft`):
+  ```bash
+  terradev lora peft import -a vineetsharma/qlora-adapter-Mistral-7B-Instruct-v0.1-gsm8k
+  terradev lora peft import -a username/adapter --local-name my-adapter --register --base-model mistralai/Mistral-7B-Instruct-v0.1
+  terradev lora peft list
+  terradev lora peft validate -p ~/.terradev/peft_adapters/username--adapter
+  terradev lora peft delete -a username/adapter
+  ```
+
+- **LoRAX Helm Template** (`clusters/lorax-template/helm/`): Production-ready Kubernetes manifests with GPU resource limits, storage configuration, and Prometheus metrics support.
+
+- **Registry Integration**: One-step import from HuggingFace and automatic registration in Terradev LoRA registry with version tracking, cross-replica sync, and cost attribution.
+
 **NOTES ON 5.3.3**
 
 Added **provider registration and profiling system** for intelligent quirk-aware routing across 23 cloud providers, and registration for custom providers from .yaml import:

{terradev_cli-5.3.8/terradev_cli → terradev_cli-5.3.9}/README.md

@@ -16,6 +16,38 @@ Continued focus on lower cost, faster provisioning, and topology-aware execution
 
 Model agnostic. Dataset agnostic. GPU agnostic. Provider agnostic. The only thing Terradev is not agnostic about is correctness: it enforces topology, idempotency, and sequencing.
 
+**NOTES ON 5.3.9**
+
+Added **LoRAX (LoRA eXchange) integration** and **HuggingFace PEFT import** for production-grade multi-LoRA inference serving:
+
+- **LoRAX Service** (`ml_services/lorax_service.py`): Async HTTP client for Predibase LoRAX multi-LoRA inference server that serves thousands of fine-tuned models on a single GPU with dynamic adapter loading, heterogeneous continuous batching, and adapter exchange scheduling.
+
+- **LoRAX CLI commands** (`terradev lora lorax`):
+  ```bash
+  terradev lora lorax deploy -m mistralai/Mistral-7B-Instruct-v0.1 --docker
+  terradev lora lorax test --host localhost --port 8080
+  terradev lora lorax list-adapters
+  terradev lora lorax load-adapter -a vineetsharma/qlora-adapter-Mistral-7B-Instruct-v0.1-gsm8k
+  terradev lora lorax unload-adapter -a my-adapter
+  terradev lora lorax generate -p "What is 2+2?" -a my-adapter
+  terradev lora lorax sync-registry
+  ```
+
+- **PEFT Import Service** (`ml_services/peft_import_service.py`): Download, validate, and prepare LoRA adapters from HuggingFace using the PEFT library with auto-detection of rank, alpha, and target modules.
+
+- **PEFT CLI commands** (`terradev lora peft`):
+  ```bash
+  terradev lora peft import -a vineetsharma/qlora-adapter-Mistral-7B-Instruct-v0.1-gsm8k
+  terradev lora peft import -a username/adapter --local-name my-adapter --register --base-model mistralai/Mistral-7B-Instruct-v0.1
+  terradev lora peft list
+  terradev lora peft validate -p ~/.terradev/peft_adapters/username--adapter
+  terradev lora peft delete -a username/adapter
+  ```
+
+- **LoRAX Helm Template** (`clusters/lorax-template/helm/`): Production-ready Kubernetes manifests with GPU resource limits, storage configuration, and Prometheus metrics support.
+
+- **Registry Integration**: One-step import from HuggingFace and automatic registration in Terradev LoRA registry with version tracking, cross-replica sync, and cost attribution.
+
 **NOTES ON 5.3.3**
 
 Added **provider registration and profiling system** for intelligent quirk-aware routing across 23 cloud providers, and registration for custom providers from .yaml import:

terradev_cli-5.3.9/clusters/lorax-template/helm/Chart.yaml

@@ -0,0 +1,17 @@
+apiVersion: v2
+name: lorax
+description: LoRAX (LoRA eXchange) multi-LoRA inference server from Predibase
+type: application
+version: 0.1.0
+appVersion: "main"
+keywords:
+  - lorax
+  - lora
+  - inference
+  - llm
+  - predibase
+home: https://github.com/predibase/lorax
+sources:
+  - https://github.com/predibase/lorax
+maintainers:
+  - name: Terradev

terradev_cli-5.3.9/clusters/lorax-template/helm/templates/_helpers.tpl

@@ -0,0 +1,29 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "lorax.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+*/}}
+{{- define "lorax.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "lorax.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}

terradev_cli-5.3.9/clusters/lorax-template/helm/templates/deployment.yaml

@@ -0,0 +1,77 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "lorax.fullname" . }}
+  labels:
+    app: {{ include "lorax.name" . }}
+    chart: {{ include "lorax.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ include "lorax.name" . }}
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: {{ include "lorax.name" . }}
+        release: {{ .Release.Name }}
+    spec:
+      serviceAccountName: {{ include "lorax.fullname" . }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+      - name: lorax
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        ports:
+        - name: http
+          containerPort: {{ .Values.service.targetPort }}
+          protocol: TCP
+        {{- if .Values.metrics.enabled }}
+        - name: metrics
+          containerPort: {{ .Values.metrics.port }}
+          protocol: TCP
+        {{- end }}
+        env:
+        {{- range .Values.env }}
+        - name: {{ .name }}
+          value: {{ .value | quote }}
+        {{- end }}
+        resources:
+          {{- toYaml .Values.resources | nindent 10 }}
+        volumeMounts:
+        {{- if .Values.storage.enabled }}
+        - name: adapter-storage
+          mountPath: /data
+        {{- end }}
+        command:
+        - lorax-server
+        args:
+        - --model-id
+        - {{ .Values.model.id }}
+        - --max-loras
+        - {{ .Values.maxLoras | quote }}
+        {{- if ne .Values.model.quantization "none" }}
+        - --quantize
+        - {{ .Values.model.quantization }}
+        {{- end }}
+        - --port
+        - {{ .Values.service.targetPort | quote }}
+        {{- if .Values.metrics.enabled }}
+        - --enable-metrics
+        {{- end }}
+      volumes:
+      {{- if .Values.storage.enabled }}
+      - name: adapter-storage
+        persistentVolumeClaim:
+          claimName: {{ include "lorax.fullname" . }}-storage
+      {{- end }}

terradev_cli-5.3.9/clusters/lorax-template/helm/templates/pvc.yaml

@@ -0,0 +1,20 @@
+{{- if .Values.storage.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ include "lorax.fullname" . }}-storage
+  labels:
+    app: {{ include "lorax.name" . }}
+    chart: {{ include "lorax.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ .Values.storage.size }}
+  {{- if .Values.storage.storageClass }}
+  storageClassName: {{ .Values.storage.storageClass }}
+  {{- end }}
+{{- end }}

terradev_cli-5.3.9/clusters/lorax-template/helm/templates/service.yaml

@@ -0,0 +1,25 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "lorax.fullname" . }}
+  labels:
+    app: {{ include "lorax.name" . }}
+    chart: {{ include "lorax.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+  - name: http
+    port: {{ .Values.service.port }}
+    targetPort: http
+    protocol: TCP
+  {{- if .Values.metrics.enabled }}
+  - name: metrics
+    port: {{ .Values.metrics.port }}
+    targetPort: metrics
+    protocol: TCP
+  {{- end }}
+  selector:
+    app: {{ include "lorax.name" . }}
+    release: {{ .Release.Name }}

terradev_cli-5.3.9/clusters/lorax-template/helm/templates/serviceaccount.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "lorax.fullname" . }}
+  labels:
+    app: {{ include "lorax.name" . }}
+    chart: {{ include "lorax.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}

terradev_cli-5.3.9/clusters/lorax-template/helm/values-lorax.yaml

@@ -0,0 +1,66 @@
+# LoRAX Helm Values
+# Deploy LoRAX multi-LoRA inference server
+
+replicaCount: 1
+
+image:
+  repository: ghcr.io/predibase/lorax
+  tag: main
+  pullPolicy: IfNotPresent
+
+# Base model configuration
+model:
+  id: mistralai/Mistral-7B-Instruct-v0.1  # HuggingFace model ID
+  quantization: none  # none, bitsandbytes, gptq, awq
+
+# LoRA adapter configuration
+maxLoras: 8  # Maximum number of adapters to load simultaneously
+
+# Service configuration
+service:
+  type: ClusterIP
+  port: 80
+  targetPort: 80
+
+# Resource limits
+resources:
+  limits:
+    nvidia.com/gpu: 1
+    memory: 32Gi
+  requests:
+    cpu: 4
+    memory: 16Gi
+
+# Storage for adapter cache
+storage:
+  enabled: true
+  size: 100Gi
+  storageClass: standard
+
+# Node selector for GPU nodes
+nodeSelector:
+  accelerator: nvidia-tesla-a100
+
+# Tolerations for GPU nodes
+tolerations:
+  - key: nvidia.com/gpu
+    operator: Exists
+    effect: NoSchedule
+
+# Environment variables
+env:
+  - name: CUDA_VISIBLE_DEVICES
+    value: "0"
+  - name: SHM_SIZE
+    value: "1g"
+
+# Prometheus metrics
+metrics:
+  enabled: true
+  port: 8080
+  path: /metrics
+
+# OpenTelemetry tracing
+tracing:
+  enabled: false
+  endpoint: http://jaeger:4318