robovast-cluster 2.1.0__tar.gz

robovast_cluster-2.1.0/PKG-INFO

@@ -0,0 +1,36 @@
+Metadata-Version: 2.1
+Name: robovast-cluster
+Version: 2.1.0
+Summary: The Kubernetes execution lane for RoboVAST: cluster campaigns, deployment, and the operator commands.
+License: Apache-2.0
+Author: Frederik Pasch
+Author-email: fred-labs@mailbox.org
+Requires-Python: >=3.12,<3.14
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: bcrypt (>=4.0,<5.0)
+Requires-Dist: boto3 (>=1.35,<2.0)
+Requires-Dist: google-cloud-storage (>=2.0,<3.0)
+Requires-Dist: kubernetes (>=34.1,<35.0)
+Requires-Dist: robovast (>=2.0.0,<3.0.0)
+Description-Content-Type: text/markdown
+
+# robovast-cluster
+
+The Kubernetes execution lane for [RoboVAST](https://github.com/cps-test-lab/robovast):
+running campaigns as Kubernetes Jobs, deploying the service, and the operator commands
+that set a cluster up and take it down.
+
+Install it alongside `robovast` when you have a cluster:
+
+```bash
+pip install robovast robovast-cluster
+```
+
+Without it, `robovast` runs the local Docker lane and says so — `vast serve --backend
+cluster` reports which lanes are installed, and `vast doctor` reports the cluster lane as
+absent rather than broken.
+
+It ships into the `robovast` namespace, so nothing about the import paths changes.
+

robovast_cluster-2.1.0/README.md

@@ -0,0 +1,17 @@
+# robovast-cluster
+
+The Kubernetes execution lane for [RoboVAST](https://github.com/cps-test-lab/robovast):
+running campaigns as Kubernetes Jobs, deploying the service, and the operator commands
+that set a cluster up and take it down.
+
+Install it alongside `robovast` when you have a cluster:
+
+```bash
+pip install robovast robovast-cluster
+```
+
+Without it, `robovast` runs the local Docker lane and says so — `vast serve --backend
+cluster` reports which lanes are installed, and `vast doctor` reports the cluster lane as
+absent rather than broken.
+
+It ships into the `robovast` namespace, so nothing about the import paths changes.

robovast_cluster-2.1.0/pyproject.toml

@@ -0,0 +1,63 @@
+[tool.poetry]
+name = "robovast-cluster"
+version = "2.1.0"
+description = "The Kubernetes execution lane for RoboVAST: cluster campaigns, deployment, and the operator commands."
+authors = ["Frederik Pasch <fred-labs@mailbox.org>"]
+license = "Apache-2.0"
+readme = "README.md"
+# Ships into the `robovast` namespace rather than a name of its own, so the import paths
+# do not change: `robovast/` and `robovast/execution/` carry no `__init__.py` in either
+# distribution, which makes them PEP 420 namespace packages and lets the two trees merge.
+packages = [{include = "robovast"}]
+
+[tool.poetry.dependencies]
+python = ">=3.12,<3.14"
+# The lane is built on the core, and the edge is safe to declare: `robovast` names
+# `robovast-nav` and `robovast-sim-roqsim` as path dependencies but never this package, so
+# there is no cycle to avoid -- which is why `robovast_sim_roqsim` cannot state its
+# equivalent and this can. It has to be stated rather than assumed: the controller image
+# installs `robovast-cluster` into the pod, and every module here imports `robovast`.
+robovast = "^2.0.0"
+kubernetes = "^34.1"
+# The built-in registry's password file. ``registry:2`` verifies with Go's bcrypt and
+# rejects anything weaker, so this is the hash or no auth at all -- see registry_deploy.
+bcrypt = "^4.0"
+boto3 = "^1.35"
+google-cloud-storage = "^2.0"
+
+# `vast serve --backend cluster`. Resolved by name, so the core never imports this package
+# to start a service -- see robovast/service/serve_backends.py.
+[tool.poetry.plugins."robovast.execution_backends"]
+cluster = "robovast.execution.cluster_execution.serve_backend:ClusterServeBackend"
+
+# The operator half of `vast cluster` -- the verbs that need a kubeconfig, an API
+# server or a cluster Secret. The `cluster` group itself belongs to robovast-client, whose
+# service-driven verbs (run/stop/stop-job/log/download-cleanup) need none of this; these
+# attach to it lazily, so `vast cluster --help` lists them without importing the
+# Kubernetes client.
+#
+# `monitor` is here rather than in the client because its kubeconfig fallback view and its
+# service view are one command chosen at runtime -- moving it would split one function's
+# body across two distributions. A client-only user gets `vast campaign wait` and the
+# web UI.
+[tool.poetry.plugins."robovast.cluster_plugins"]
+monitor = "robovast.execution.cluster_execution.cli:monitor"
+setup = "robovast.execution.cluster_execution.cli:setup"
+cleanup = "robovast.execution.cluster_execution.cli:cleanup"
+"jobs-cleanup" = "robovast.execution.cluster_execution.cli:run_cleanup"
+
+# `vast service upgrade` / `token` act on the deployed SERVICE, not on the cluster, so they
+# attach to that group -- and need an API server, so they attach from here.
+[tool.poetry.plugins."robovast.service_plugins"]
+upgrade = "robovast.execution.cluster_execution.cli:upgrade"
+token = "robovast.execution.cluster_execution.cli:cluster_token"
+
+[tool.poetry.plugins."robovast.cluster_configs"]
+minikube = "robovast.execution.cluster_config.minikube:MinikubeClusterConfig"
+rke2 = "robovast.execution.cluster_config.rke2:Rke2ClusterConfig"
+gcp = "robovast.execution.cluster_config.gcp:GcpClusterConfig"
+azure = "robovast.execution.cluster_config.azure:AzureClusterConfig"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

robovast_cluster-2.1.0/robovast/execution/cluster_config/azure.py

@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+# Copyright (C) 2025 Frederik Pasch
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions
+# and limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+import io
+import logging
+
+import yaml
+from kubernetes import client
+
+from robovast.execution.cluster_execution.kube_client import load_kube_config
+
+from ..cluster_execution import store_pod
+from ..cluster_execution.kubernetes import apply_manifests, delete_manifests
+from .base_config import BaseConfig
+
+MINIO_MANIFEST_AZURE = """---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: robovast-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: {storage_size}
+  storageClassName: managed-csi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: robovast
+  labels:
+    role: robovast
+spec:
+  containers:
+  - name: minio
+    image: minio/minio:latest
+    args: ["server", "/data", "--console-address", ":9001"]
+    env:
+    - name: MINIO_ROOT_USER
+      value: "minioadmin"
+    - name: MINIO_ROOT_PASSWORD
+      value: "minioadmin"
+    ports:
+    - name: s3
+      containerPort: 9000
+    - name: console
+      containerPort: 9001
+    volumeMounts:
+    - mountPath: /data
+      name: minio-storage
+    resources:
+      # A floor, not an estimate: every milli-core requested here is capacity campaign jobs
+      # cannot be admitted against, so the request covers an idle server. The LIMIT is a
+      # different quantity and has to be sized against the peak: MinIO streams objects, but
+      # a batch mirrors dozens of multi-part transfers at once, and each carries buffers the
+      # request knows nothing about. Sized well above idle on purpose -- a store that is
+      # OOM-killed mid-batch takes that batch's bag conversion with it, and the campaign then
+      # fails in postprocessing with no conversion log to explain it. No CPU limit --
+      # throttling the store slows every upload and download with no visible cause.
+      requests:
+        cpu: "50m"
+        memory: "256Mi"
+      limits:
+        memory: "4Gi"
+    readinessProbe:
+      httpGet:
+        path: /minio/health/ready
+        port: 9000
+      initialDelaySeconds: 10
+      periodSeconds: 5
+  volumes:
+  - name: minio-storage
+    persistentVolumeClaim:
+      claimName: robovast-pvc
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: robovast
+spec:
+  ports:
+  - name: s3
+    port: 9000
+    targetPort: 9000
+  - name: console
+    port: 9001
+    targetPort: 9001
+  selector:
+    role: robovast
+"""
+
+
+class AzureClusterConfig(BaseConfig):
+
+    #: AKS node pools are Azure VMs, whose guest kernels expose no cpufreq policy.
+    #: See :attr:`BaseConfig.governor_is_settable`.
+    governor_is_settable = False
+
+    def setup_cluster(self, storage_size="10Gi", **kwargs):
+        """Set up MinIO S3 server for Azure cluster.
+
+        Args:
+            storage_size (str): Size of the persistent volume (default: "10Gi")
+            **kwargs: Additional cluster-specific options (kube_context, namespace,
+                control_node_labels)
+        """
+        control_node_labels = kwargs.pop('control_node_labels', None)
+        logging.info("Setting up RoboVAST MinIO S3 server in Azure cluster...")
+        logging.info(f"Storage size: {storage_size}")
+
+        load_kube_config(context=kwargs.get('kube_context'))
+        k8s_client = client.ApiClient()
+
+        try:
+            yaml_objects = list(yaml.safe_load_all(io.StringIO(MINIO_MANIFEST_AZURE.format(storage_size=storage_size))))
+        except yaml.YAMLError as e:
+            raise RuntimeError(f"Failed to parse MinIO manifest YAML: {str(e)}") from e
+
+        namespace = kwargs.get('namespace', 'default')
+        # The registry and the campaign index ride in this pod: setup-lifetime
+        # infrastructure, created once here rather than in the service Deployment that
+        # every upgrade rolls. See `cluster_execution.store_pod`.
+        yaml_objects = store_pod.attach_infrastructure(
+            yaml_objects, namespace,
+            index_storage_path=kwargs.get('index_storage_path', ''),
+            index_storage_class=kwargs.get('index_storage_class', ''),
+            index_storage_size=kwargs.get('index_storage_size', ''),
+            registry_storage_path=kwargs.get('registry_storage_path', ''),
+            registry_storage_class=kwargs.get('registry_storage_class', ''),
+            registry_authenticated=kwargs.get('registry_authenticated', False),
+            ingress_class=kwargs.get('ingress_class', ''))
+        yaml_objects = self._apply_pod_node_selector(yaml_objects, control_node_labels)
+        try:
+            apply_manifests(k8s_client, iter(yaml_objects), namespace=namespace)
+        except Exception as e:
+            raise RuntimeError(f"Error applying MinIO manifest: {str(e)}") from e
+
+        logging.info(f"MinIO S3 server available at: {self.get_s3_endpoint()}")
+
+    def cleanup_cluster(self, storage_size="10Gi", **kwargs):
+        """Clean up MinIO S3 server for Azure cluster.
+
+        Args:
+            storage_size (str): Size of the persistent volume (default: "10Gi")
+            **kwargs: Additional cluster-specific options (ignored)
+        """
+        logging.debug("Cleaning up RoboVAST MinIO in Azure cluster...")
+        load_kube_config(context=kwargs.get('kube_context'))
+        core_v1 = client.CoreV1Api()
+
+        try:
+            yaml_objects = yaml.safe_load_all(io.StringIO(MINIO_MANIFEST_AZURE.format(storage_size=storage_size)))
+        except yaml.YAMLError as e:
+            raise RuntimeError(f"Failed to parse MinIO manifest YAML: {str(e)}") from e
+
+        namespace = kwargs.get('namespace', 'default')
+        yaml_objects = store_pod.attach_infrastructure(list(yaml_objects), namespace)
+        delete_manifests(core_v1, store_pod.infrastructure_claims(namespace) + yaml_objects,
+                         namespace=namespace)
+        logging.debug("MinIO manifest deleted successfully!")
+
+    def prepare_setup_cluster(self, output_dir, storage_size="10Gi", **kwargs):
+        """Prepare any prerequisites before setting up the cluster.
+
+        Args:
+            output_dir (str): Directory where setup files will be written
+            storage_size (str): Size of the persistent volume (default: "10Gi")
+            **kwargs: Cluster-specific options (control_node_labels)
+        """
+        control_node_labels = kwargs.pop('control_node_labels', None)
+        docs = store_pod.attach_infrastructure(
+            list(yaml.safe_load_all(io.StringIO(
+                MINIO_MANIFEST_AZURE.format(storage_size=storage_size)))),
+            kwargs.get('namespace', 'default'))
+        docs = self._apply_pod_node_selector(docs, control_node_labels)
+        manifest_to_write = "---\n".join(
+            yaml.dump(d, default_flow_style=False) for d in docs if d is not None
+        )
+        with open(f"{output_dir}/robovast-manifest.yaml", "w") as f:
+            f.write(manifest_to_write)
+
+        readme_content = f"""# Azure Cluster Setup Instructions
+
+Uses MinIO backed by an Azure managed-csi PVC ({storage_size}).
+
+## Setup Steps
+
+### 1. Apply the RoboVAST MinIO Manifest
+
+```bash
+kubectl apply -f robovast-manifest.yaml
+```
+
+### 2. Wait for the pod to be ready
+
+```bash
+kubectl wait --for=condition=ready pod/robovast --timeout=120s
+```
+
+MinIO S3 API is available at `http://robovast:9000` (cluster-internal).
+MinIO console is available at port 9001.
+"""
+        with open(f"{output_dir}/README_azure.md", "w") as f:
+            f.write(readme_content)
+
+    def get_instance_type_command(self):
+        """Get command to retrieve instance type of the current node."""
+        return (
+            'INSTANCE_TYPE=$(curl -s -H "Metadata: true" '
+            '"http://169.254.169.254/metadata/instance/compute/vmSize'
+            '?api-version=2021-02-01&format=text")'
+        )