argus-alm 0.14.2__py3-none-any.whl → 0.15.2__py3-none-any.whl

argus/backend/plugins/generic/model.py

@@ -0,0 +1,86 @@
+from datetime import datetime
+import re
+from uuid import UUID
+from cassandra.cqlengine import columns
+from cassandra.cqlengine.models import Model
+from argus.backend.db import ScyllaCluster
+from argus.backend.models.web import ArgusRelease
+from argus.backend.plugins.core import PluginModelBase
+from argus.backend.plugins.generic.types import GenericRunFinishRequest, GenericRunSubmitRequest
+from argus.common.enums import TestStatus
+
+
+class GenericPluginException(Exception):
+    pass
+
+
+class GenericRun(PluginModelBase):
+    _plugin_name = "generic"
+    __table_name__ = "generic_run"
+    logs = columns.Map(key_type=columns.Text(), value_type=columns.Text())
+    started_by = columns.Text()
+    # TODO: Legacy field name, should be renamed to product_version and abstracted
+    scylla_version = columns.Text()
+
+    @classmethod
+    def _stats_query(cls) -> str:
+        return ("SELECT id, test_id, group_id, release_id, status, start_time, build_job_url, build_id, "
+                f"assignee, end_time, investigation_status, heartbeat, scylla_version FROM {cls.table_name()} WHERE build_id IN ? PER PARTITION LIMIT 15")
+
+    @classmethod
+    def get_distinct_product_versions(cls, release: ArgusRelease, cluster: ScyllaCluster = None) -> list[str]:
+        if not cluster:
+            cluster = ScyllaCluster.get()
+        statement = cluster.prepare(f"SELECT scylla_version FROM {cls.table_name()} WHERE release_id = ?")
+        rows = cluster.session.execute(query=statement, parameters=(release.id,))
+        unique_versions = {r["scylla_version"] for r in rows if r["scylla_version"]}
+
+        return sorted(list(unique_versions), reverse=True)
+
+    def submit_product_version(self, version: str):
+        pattern = re.compile(r"((?P<short>[\w.~]+)-(?P<build>(0\.)?(?P<date>[0-9]{8,8})\.(?P<commit>\w+).*))")
+        if match := pattern.search(version):
+            self.scylla_version = match.group("short")
+            try:
+                new_assignee = self.get_assignment(match.group("short"))
+            except Model.DoesNotExist:
+                new_assignee = None
+            if new_assignee:
+                self.assignee = new_assignee
+            self.set_full_version(version)
+
+    @classmethod
+    def load_test_run(cls, run_id: UUID) -> 'GenericRun':
+        return cls.get(id=run_id)
+
+    @classmethod
+    def submit_run(cls, request_data: GenericRunSubmitRequest) -> 'GenericRun':
+        try:
+            run = cls.get(id=request_data["run_id"])
+            raise GenericPluginException(
+                f"Run with UUID {request_data['run_id']} already exists.", request_data["run_id"])
+        except cls.DoesNotExist:
+            pass
+        run = cls()
+        run.start_time = datetime.utcnow()
+        run.build_id = request_data["build_id"]
+        run.started_by = request_data["started_by"]
+        run.id = request_data["run_id"]
+        run.build_job_url = request_data["build_url"]
+        run.assign_categories()
+        try:
+            run.assignee = run.get_scheduled_assignee()
+        except Model.DoesNotExist:
+            run.assignee = None
+        if version := request_data.get("scylla_version"):
+            run.submit_product_version(version)
+        run.status = TestStatus.RUNNING.value
+        run.save()
+
+        return run
+
+    def finish_run(self, payload: GenericRunFinishRequest = None):
+        self.end_time = datetime.utcnow()
+        self.status = TestStatus(payload["status"]).value
+        if version := payload.get("scylla_version"):
+            self.submit_product_version(version)

argus/backend/plugins/generic/plugin.py

@@ -0,0 +1,15 @@
+from flask import Blueprint
+
+from argus.backend.plugins.core import PluginInfoBase, PluginModelBase
+from argus.backend.plugins.generic.model import GenericRun
+
+
+class PluginInfo(PluginInfoBase):
+    name: str = "generic"
+    model: PluginModelBase = GenericRun
+    controller: Blueprint = None
+    all_models = [
+        GenericRun
+    ]
+    all_types = [
+    ]

argus/backend/plugins/generic/types.py

@@ -0,0 +1,14 @@
+from typing import TypedDict
+
+
+class GenericRunSubmitRequest(TypedDict):
+    build_id: str
+    build_url: str
+    run_id: str
+    started_by: str
+    scylla_version: str | None
+
+
+class GenericRunFinishRequest(TypedDict):
+    status: str
+    scylla_version: str | None

argus/backend/plugins/loader.py

@@ -0,0 +1,39 @@
+from pathlib import Path
+import typing
+from argus.backend.plugins.core import PluginInfoBase
+from argus.backend.plugins.core import PluginModelBase
+
+
+class PluginModule(typing.Protocol):
+    PluginInfo: PluginInfoBase
+
+
+def plugin_loader() -> dict[str, PluginInfoBase]:
+    loader_path = Path(__file__).parent
+    plugin_dirs = [p for p in loader_path.glob("*") if p.is_dir()]
+    modules = {}
+    parent_module = ".".join(__name__.split(".")[:-1])
+    for directory in plugin_dirs:
+        if (plugin_path := directory / "plugin.py").exists():
+            rel_path = str(plugin_path.relative_to(loader_path))
+            module_path = rel_path.replace("/", ".").replace(".py", "")
+            module: PluginModule = __import__(
+                f"{parent_module}.{module_path}",
+                globals=globals(),
+                fromlist=("PluginInfo",)
+            )
+            plugin = module.PluginInfo
+            modules[plugin.name] = plugin
+
+    return modules
+
+
+AVAILABLE_PLUGINS = plugin_loader()
+
+
+def all_plugin_models(include_all=False) -> list[PluginModelBase]:
+    return [model for plugin in AVAILABLE_PLUGINS.values() for model in plugin.all_models if issubclass(model, PluginModelBase) or include_all]
+
+
+def all_plugin_types():
+    return [user_type for plugin in AVAILABLE_PLUGINS.values() for user_type in plugin.all_types]

argus/backend/plugins/sct/controller.py

@@ -0,0 +1,224 @@
+from flask import Blueprint, request
+
+from argus.backend.error_handlers import handle_api_exception
+from argus.backend.service.user import api_login_required
+from argus.backend.plugins.sct.service import SCTService
+from argus.backend.util.common import get_payload
+
+bp = Blueprint("sct_api", __name__, url_prefix="/sct")
+bp.register_error_handler(Exception, handle_api_exception)
+
+
+@bp.route("/<string:run_id>/packages/submit", methods=["POST"])
+@api_login_required
+def sct_submit_packages(run_id: str):
+    payload = get_payload(request)
+    result = SCTService.submit_packages(run_id=run_id, packages=payload["packages"])
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/<string:run_id>/screenshots/submit", methods=["POST"])
+@api_login_required
+def sct_submit_screenshots(run_id: str):
+    payload = get_payload(request)
+    result = SCTService.submit_screenshots(run_id=run_id, screenshot_links=payload["screenshot_links"])
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/<string:run_id>/sct_runner/set", methods=["POST"])
+@api_login_required
+def sct_set_runner(run_id: str):
+    payload = get_payload(request)
+    result = SCTService.set_sct_runner(
+        run_id=run_id,
+        public_ip=payload["public_ip"],
+        private_ip=payload["private_ip"],
+        region=payload["region"],
+        backend=payload["backend"],
+        name=payload.get("name")
+    )
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/<string:run_id>/resource/create", methods=["POST"])
+@api_login_required
+def sct_resource_create(run_id: str):
+    payload = get_payload(request)
+    result = SCTService.create_resource(run_id=run_id, resource_details=payload["resource"])
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/<string:run_id>/resource/<string:resource_name>/terminate", methods=["POST"])
+@api_login_required
+def sct_resource_terminate(run_id: str, resource_name: str):
+    payload = get_payload(request)
+    result = SCTService.terminate_resource(run_id=run_id, resource_name=resource_name, reason=payload["reason"])
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/<string:run_id>/resource/<string:resource_name>/shards", methods=["POST"])
+@api_login_required
+def sct_resource_update_shards(run_id: str, resource_name: str):
+    payload = get_payload(request)
+    result = SCTService.update_resource_shards(run_id=run_id, resource_name=resource_name, new_shards=payload["shards"])
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/<string:run_id>/resource/<string:resource_name>/update", methods=["POST"])
+@api_login_required
+def sct_resource_update(run_id: str, resource_name: str):
+    payload = get_payload(request)
+    result = SCTService.update_resource(run_id=run_id, resource_name=resource_name, update_data=payload["update_data"])
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/<string:run_id>/nemesis/submit", methods=["POST"])
+@api_login_required
+def sct_nemesis_submit(run_id: str):
+    payload = get_payload(request)
+    result = SCTService.submit_nemesis(run_id=run_id, nemesis_details=payload["nemesis"])
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/<string:run_id>/nemesis/finalize", methods=["POST"])
+@api_login_required
+def sct_nemesis_finalize(run_id: str):
+    payload = get_payload(request)
+    result = SCTService.finalize_nemesis(run_id=run_id, nemesis_details=payload["nemesis"])
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/<string:run_id>/events/submit", methods=["POST"])
+@api_login_required
+def sct_events_submit(run_id: str):
+    payload = get_payload(request)
+    result = SCTService.submit_events(run_id=run_id, events=payload["events"])
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/<string:run_id>/gemini/submit", methods=["POST"])
+@api_login_required
+def sct_gemini_results_submit(run_id: str):
+    payload = get_payload(request)
+    result = SCTService.submit_gemini_results(run_id=run_id, gemini_data=payload["gemini_data"])
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/<string:run_id>/performance/submit", methods=["POST"])
+@api_login_required
+def sct_performance_results_submit(run_id: str):
+    payload = get_payload(request)
+    result = SCTService.submit_performance_results(run_id=run_id, performance_results=payload["performance_results"])
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/<string:run_id>/performance/history", methods=["GET"])
+@api_login_required
+def sct_get_performance_history(run_id: str):
+    result = SCTService.get_performance_history_for_test(run_id=run_id)
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/release/<string:release_name>/kernels", methods=["GET"])
+@api_login_required
+def sct_get_kernel_report(release_name: str):
+    result = SCTService.get_scylla_version_kernels_report(release_name=release_name)
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/<string:run_id>/junit/submit", methods=["POST"])
+@api_login_required
+def sct_submit_junit_report(run_id: str):
+    payload = get_payload(request)
+    result = SCTService.junit_submit(run_id, payload["file_name"], payload["content"])
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/<string:run_id>/junit/get_all", methods=["GET"])
+@api_login_required
+def sct_get_junit_reports(run_id: str):
+    result = SCTService.junit_get_all(run_id)
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/<string:run_id>/similar_events", methods=["GET"])
+@api_login_required
+def sct_get_similar_events(run_id: str):
+    result = SCTService.get_similar_events(run_id=run_id)
+    return {
+        "status": "ok",
+        "response": result
+    }
+
+
+@bp.route("/similar_runs_info", methods=["POST"])
+@api_login_required
+def sct_get_similar_runs_info():
+    """Get build IDs and issues for a list of run IDs"""
+    data = request.get_json()
+    if not data or "run_ids" not in data:
+        return {
+            "status": "error",
+            "response": "Missing run_ids parameter"
+        }, 400
+
+    run_ids = data["run_ids"]
+    if not isinstance(run_ids, list):
+        return {
+            "status": "error",
+            "response": "run_ids must be a list"
+        }, 400
+
+    result = SCTService.get_similar_runs_info(run_ids=run_ids)
+    return {
+        "status": "ok",
+        "response": result
+    }

argus/backend/plugins/sct/plugin.py

@@ -0,0 +1,37 @@
+from flask import Blueprint
+
+from argus.backend.plugins.sct.testrun import SCTJunitReports, SCTTestRun
+from argus.backend.plugins.sct.controller import bp as sct_bp
+from argus.backend.plugins.core import PluginInfoBase, PluginModelBase
+from argus.backend.plugins.sct.udt import (
+    CloudInstanceDetails,
+    CloudNodesInfo,
+    CloudResource,
+    CloudSetupDetails,
+    EventsBySeverity,
+    NemesisRunInfo,
+    NodeDescription,
+    PackageVersion,
+    PerformanceHDRHistogram,
+)
+
+
+class PluginInfo(PluginInfoBase):
+    name: str = "scylla-cluster-tests"
+    model: PluginModelBase = SCTTestRun
+    controller: Blueprint = sct_bp
+    all_models = [
+        SCTTestRun,
+        SCTJunitReports,
+    ]
+    all_types = [
+        NemesisRunInfo,
+        NodeDescription,
+        EventsBySeverity,
+        CloudResource,
+        CloudSetupDetails,
+        CloudNodesInfo,
+        CloudInstanceDetails,
+        PackageVersion,
+        PerformanceHDRHistogram,
+    ]

argus/backend/plugins/sct/resource_setup.py

@@ -0,0 +1,177 @@
+import logging
+from argus.backend.plugins.sct.udt import CloudNodesInfo, CloudSetupDetails
+
+LOGGER = logging.getLogger(__name__)
+
+
+def _get_node_amounts(config: dict) -> tuple[int, int]:
+    num_db_node = config.get("n_db_nodes")
+    num_db_node = sum([int(i) for i in num_db_node.split()]) if isinstance(num_db_node, str) else num_db_node
+    num_loaders = config.get("n_loaders")
+    num_loaders = sum([int(i) for i in num_loaders.split()]) if isinstance(num_loaders, str) else num_loaders
+
+    return num_db_node, num_loaders
+
+
+def _prepare_aws_resource_setup(sct_config: dict) -> CloudSetupDetails:
+    num_db_nodes, n_loaders = _get_node_amounts(sct_config)
+    db_node_setup = CloudNodesInfo(image_id=sct_config.get("ami_id_db_scylla"),
+                                   instance_type=sct_config.get("instance_type_db"),
+                                   node_amount=num_db_nodes,
+                                   post_behaviour=sct_config.get("post_behavior_db_nodes"))
+    loader_node_setup = CloudNodesInfo(image_id=sct_config.get("ami_id_loader"),
+                                       instance_type=sct_config.get("instance_type_loader"),
+                                       node_amount=n_loaders,
+                                       post_behaviour=sct_config.get("post_behavior_loader_nodes"))
+    monitor_node_setup = CloudNodesInfo(image_id=sct_config.get("ami_id_monitor"),
+                                        instance_type=sct_config.get("instance_type_monitor"),
+                                        node_amount=sct_config.get("n_monitor_nodes"),
+                                        post_behaviour=sct_config.get("post_behavior_monitor_nodes"))
+    cloud_setup = CloudSetupDetails(db_node=db_node_setup, loader_node=loader_node_setup,
+                                    monitor_node=monitor_node_setup, backend=sct_config.get("cluster_backend"))
+
+    return cloud_setup
+
+
+def _prepare_gce_resource_setup(sct_config: dict) -> CloudSetupDetails:
+    num_db_nodes, n_loaders = _get_node_amounts(sct_config)
+    db_node_setup = CloudNodesInfo(image_id=sct_config.get("gce_image_db"),
+                                   instance_type=sct_config.get("gce_instance_type_db"),
+                                   node_amount=num_db_nodes,
+                                   post_behaviour=sct_config.get("post_behavior_db_nodes"))
+    loader_node_setup = CloudNodesInfo(image_id=sct_config.get("gce_image_loader"),
+                                       instance_type=sct_config.get("gce_instance_type_loader"),
+                                       node_amount=n_loaders,
+                                       post_behaviour=sct_config.get("post_behavior_loader_nodes"))
+    monitor_node_setup = CloudNodesInfo(image_id=sct_config.get("gce_image_monitor"),
+                                        instance_type=sct_config.get("gce_instance_type_monitor"),
+                                        node_amount=sct_config.get("n_monitor_nodes"),
+                                        post_behaviour=sct_config.get("post_behavior_monitor_nodes"))
+    cloud_setup = CloudSetupDetails(db_node=db_node_setup, loader_node=loader_node_setup,
+                                    monitor_node=monitor_node_setup, backend=sct_config.get("cluster_backend"))
+
+    return cloud_setup
+
+
+def _prepare_azure_resource_setup(sct_config: dict) -> CloudSetupDetails:
+    num_db_nodes, n_loaders = _get_node_amounts(sct_config)
+    db_node_setup = CloudNodesInfo(image_id=sct_config.get("azure_image_db"),
+                                   instance_type=sct_config.get("azure_instance_type_db"),
+                                   node_amount=num_db_nodes,
+                                   post_behaviour=sct_config.get("post_behavior_db_nodes"))
+    loader_node_setup = CloudNodesInfo(image_id=sct_config.get("azure_image_loader"),
+                                       instance_type=sct_config.get("azure_instance_type_loader"),
+                                       node_amount=n_loaders,
+                                       post_behaviour=sct_config.get("post_behavior_loader_nodes"))
+    monitor_node_setup = CloudNodesInfo(image_id=sct_config.get("azure_image_monitor"),
+                                        instance_type=sct_config.get("azure_instance_type_monitor"),
+                                        node_amount=sct_config.get("n_monitor_nodes"),
+                                        post_behaviour=sct_config.get("post_behavior_monitor_nodes"))
+    cloud_setup = CloudSetupDetails(db_node=db_node_setup, loader_node=loader_node_setup,
+                                    monitor_node=monitor_node_setup, backend=sct_config.get("cluster_backend"))
+
+    return cloud_setup
+
+
+def _prepare_unknown_resource_setup(sct_config: dict) -> CloudSetupDetails:
+    LOGGER.error("Unknown backend encountered: %s", sct_config.get("cluster_backend"))
+    db_node_setup = CloudNodesInfo(image_id="UNKNOWN",
+                                   instance_type="UNKNOWN",
+                                   node_amount=-1,
+                                   post_behaviour="UNKNOWN")
+    loader_node_setup = CloudNodesInfo(image_id="UNKNOWN",
+                                       instance_type="UNKNOWN",
+                                       node_amount=-1,
+                                       post_behaviour="UNKNOWN")
+    monitor_node_setup = CloudNodesInfo(image_id="UNKNOWN",
+                                        instance_type="UNKNOWN",
+                                        node_amount=-1,
+                                        post_behaviour="UNKNOWN")
+    cloud_setup = CloudSetupDetails(db_node=db_node_setup, loader_node=loader_node_setup,
+                                    monitor_node=monitor_node_setup, backend=sct_config.get("cluster_backend"))
+
+    return cloud_setup
+
+
+def _prepare_bare_metal_resource_setup(sct_config: dict) -> CloudSetupDetails:
+    db_node_setup = CloudNodesInfo(image_id="bare_metal",
+                                   instance_type="bare_metal",
+                                   node_amount=sct_config.get("n_db_nodes"),
+                                   post_behaviour=sct_config.get("post_behavior_db_nodes"))
+    loader_node_setup = CloudNodesInfo(image_id="bare_metal",
+                                       instance_type="bare_metal",
+                                       node_amount=sct_config.get("n_loaders"),
+                                       post_behaviour=sct_config.get("post_behavior_loader_nodes"))
+    monitor_node_setup = CloudNodesInfo(image_id="bare_metal",
+                                        instance_type="bare_metal",
+                                        node_amount=sct_config.get("n_monitor_nodes"),
+                                        post_behaviour=sct_config.get("post_behavior_monitor_nodes"))
+    cloud_setup = CloudSetupDetails(db_node=db_node_setup, loader_node=loader_node_setup,
+                                    monitor_node=monitor_node_setup, backend=sct_config.get("cluster_backend"))
+
+    return cloud_setup
+
+
+def _prepare_k8s_gce_minikube_resource_setup(sct_config: dict) -> CloudSetupDetails:
+    cloud_setup = _prepare_gce_resource_setup(sct_config)
+
+    image_id = sct_config.get("scylla_version")
+    cloud_setup.db_node.image_id = f"scylladb/scylladb:{image_id}"
+    cloud_setup.db_node.instance_type = sct_config.get("gce_instance_type_minikube")
+
+    return cloud_setup
+
+
+def _prepare_k8s_gke_resource_setup(sct_config: dict) -> CloudSetupDetails:
+    cloud_setup = _prepare_gce_resource_setup(sct_config)
+    image_id = sct_config.get("scylla_version")
+    cloud_setup.db_node.image_id = f"scylladb/scylladb:{image_id}"
+    cloud_setup.monitor_node.image_id = sct_config.get("mgmt_docker_image")
+    cloud_setup.loader_node.image_id = f"scylladb/scylladb:{image_id}"
+
+    return cloud_setup
+
+
+def _prepare_k8s_eks_resource_setup(sct_config: dict) -> CloudSetupDetails:
+    cloud_setup = _prepare_aws_resource_setup(sct_config)
+
+    return cloud_setup
+
+
+def _prepare_docker_resource_setup(sct_config: dict) -> CloudSetupDetails:
+    db_node_setup = CloudNodesInfo(image_id=sct_config.get('docker_image'),
+                                   instance_type="docker",
+                                   node_amount=sct_config.get("n_db_nodes"),
+                                   post_behaviour=sct_config.get("post_behavior_db_nodes"))
+    loader_node_setup = CloudNodesInfo(image_id=sct_config.get('docker_image'),
+                                       instance_type="docker",
+                                       node_amount=sct_config.get("n_loaders"),
+                                       post_behaviour=sct_config.get("post_behavior_loader_nodes"))
+    monitor_node_setup = CloudNodesInfo(image_id=sct_config.get('docker_image'),
+                                        instance_type="docker",
+                                        node_amount=sct_config.get("n_monitor_nodes"),
+                                        post_behaviour=sct_config.get("post_behavior_monitor_nodes"))
+    cloud_setup = CloudSetupDetails(db_node=db_node_setup, loader_node=loader_node_setup,
+                                    monitor_node=monitor_node_setup, backend=sct_config.get("cluster_backend"))
+
+    return cloud_setup
+
+
+class ResourceSetup:
+    BACKEND_MAP = {
+        "aws": _prepare_aws_resource_setup,
+        "aws-siren": _prepare_aws_resource_setup,
+        "azure": _prepare_azure_resource_setup,
+        "gce": _prepare_gce_resource_setup,
+        "gce-siren": _prepare_gce_resource_setup,
+        "k8s-eks": _prepare_k8s_eks_resource_setup,
+        "k8s-gke": _prepare_k8s_gke_resource_setup,
+        "k8s-gce-minikube": _prepare_k8s_gce_minikube_resource_setup,
+        "baremetal": _prepare_bare_metal_resource_setup,
+        "docker": _prepare_docker_resource_setup,
+        "unknown": _prepare_unknown_resource_setup,
+    }
+
+    @classmethod
+    def get_resource_setup(cls, backend: str, sct_config: dict) -> CloudSetupDetails:
+        return cls.BACKEND_MAP.get(backend, _prepare_unknown_resource_setup)(sct_config)