argus-alm 0.14.2__py3-none-any.whl → 0.15.2__py3-none-any.whl

argus/backend/plugins/sct/service.py

@@ -0,0 +1,682 @@
+import base64
+from dataclasses import dataclass
+from datetime import datetime
+import logging
+import math
+import re
+from time import time
+from xml.etree import ElementTree
+from flask import g
+from argus.backend.models.github_issue import GithubIssue, IssueLink
+from argus.backend.models.web import ArgusEventTypes, ErrorEventEmbeddings, CriticalEventEmbeddings
+from argus.backend.plugins.sct.testrun import SCTJunitReports, SCTTestRun, SubtestType
+from argus.common.sct_types import GeminiResultsRequest, PerformanceResultsRequest, ResourceUpdateRequest
+from argus.backend.plugins.sct.udt import (
+    CloudInstanceDetails,
+    CloudResource,
+    EventsBySeverity,
+    NemesisRunInfo,
+    NodeDescription,
+    PackageVersion,
+    PerformanceHDRHistogram,
+)
+from argus.backend.service.event_service import EventService
+from argus.backend.util.common import chunk, get_build_number
+from argus.common.enums import NemesisStatus, ResourceState, TestStatus
+
+LOGGER = logging.getLogger(__name__)
+MAX_SIMILARS = 20
+
+
+class SCTServiceException(Exception):
+    pass
+
+
+@dataclass(init=True, repr=True)
+class NemesisSubmissionRequest:
+    name: str
+    class_name: str
+    start_time: int
+    node_name: str
+    node_ip: str
+    node_shards: int
+
+
+@dataclass(init=True, repr=True)
+class NemesisFinalizationRequest:
+    name: str
+    start_time: int
+    status: str
+    message: str
+
+
+@dataclass(init=True, repr=True)
+class EventSubmissionRequest:
+    severity: str
+    total_events: int
+    messages: list[str]
+
+
+class SCTService:
+
+    @staticmethod
+    def submit_packages(run_id: str, packages: list[dict]) -> str:
+        try:
+            run: SCTTestRun = SCTTestRun.get(id=run_id)
+            for package_dict in packages:
+                package = PackageVersion(**package_dict)
+                if "target" in package.name:
+                    SCTService.process_target_version(run, package)
+                if package not in run.packages:
+                    run.packages.append(package)
+            run.save()
+        except SCTTestRun.DoesNotExist as exception:
+            LOGGER.error("Run %s not found for SCTTestRun", run_id)
+            raise SCTServiceException("Run not found", run_id) from exception
+
+        return "added"
+
+    @staticmethod
+    def process_target_version(run: SCTTestRun, package: PackageVersion):
+        if run.version_source and "upgrade-target" in run.version_source and package.name == "scylla-server-target":
+            return
+        run.version_source = package.name
+        run.scylla_version = package.version
+
+    @staticmethod
+    def set_sct_runner(run_id: str, public_ip: str, private_ip: str, region: str, backend: str, name: str = None):
+        try:
+            run: SCTTestRun = SCTTestRun.get(id=run_id)
+            details = CloudInstanceDetails(
+                public_ip=public_ip,
+                private_ip=private_ip,
+                provider=backend,
+                region=region,
+            )
+            run.sct_runner_host = details
+            resource = CloudResource(name=name or "sct-runner", resource_type="sct-runner", instance_info=details)
+            run.allocated_resources.append(resource)
+            run.save()
+        except SCTTestRun.DoesNotExist as exception:
+            LOGGER.error("Run %s not found for SCTTestRun", run_id)
+            raise SCTServiceException("Run not found", run_id) from exception
+
+        return "updated"
+
+    @staticmethod
+    def submit_screenshots(run_id: str, screenshot_links: list[str]) -> str:
+        try:
+            run: SCTTestRun = SCTTestRun.get(id=run_id)
+            for link in screenshot_links:
+                run.add_screenshot(link)
+            run.save()
+        except SCTTestRun.DoesNotExist as exception:
+            LOGGER.error("Run %s not found for SCTTestRun", run_id)
+            raise SCTServiceException("Run not found", run_id) from exception
+
+        return "submitted"
+
+    @staticmethod
+    def submit_gemini_results(run_id: str, gemini_data: GeminiResultsRequest) -> str:
+        try:
+            run: SCTTestRun = SCTTestRun.get(id=run_id)
+            run.subtest_name = SubtestType.GEMINI.value
+            run.oracle_nodes_count = gemini_data.get("oracle_nodes_count")
+            run.oracle_node_ami_id = gemini_data.get("oracle_node_ami_id")
+            run.oracle_node_instance_type = gemini_data.get("oracle_node_instance_type")
+            run.oracle_node_scylla_version = gemini_data.get("oracle_node_scylla_version")
+            run.gemini_command = gemini_data.get("gemini_command")
+            run.gemini_version = gemini_data.get("gemini_version")
+            run.gemini_status = gemini_data.get("gemini_status")
+            run.gemini_seed = str(gemini_data.get("gemini_seed"))
+            run.gemini_write_ops = gemini_data.get("gemini_write_ops")
+            run.gemini_write_errors = gemini_data.get("gemini_write_errors")
+            run.gemini_read_ops = gemini_data.get("gemini_read_ops")
+            run.gemini_read_errors = gemini_data.get("gemini_read_errors")
+            run.save()
+
+            if run.gemini_status != "PASSED":
+                run.status = TestStatus.FAILED
+                EventService.create_run_event(kind=ArgusEventTypes.TestRunStatusChanged, body={
+                    "message": "[{username}] Setting run status to {status} due to Gemini reporting following status: {gemini_status}",
+                    "username": g.user.username,
+                    "status": TestStatus.FAILED.value,
+                    "gemini_status": run.gemini_status,
+                }, user_id=g.user.id, run_id=run_id, release_id=run.release_id, test_id=run.test_id)
+                run.save()
+        except SCTTestRun.DoesNotExist as exception:
+            LOGGER.error("Run %s not found for SCTTestRun", run_id)
+            raise SCTServiceException("Run not found", run_id) from exception
+
+        return "submitted"
+
+    @staticmethod
+    def submit_performance_results(run_id: str, performance_results: PerformanceResultsRequest):
+        try:
+            run: SCTTestRun = SCTTestRun.get(id=run_id)
+            run.subtest_name = SubtestType.PERFORMANCE.value
+            run.perf_op_rate_average = performance_results.get("perf_op_rate_average")
+            run.perf_op_rate_total = performance_results.get("perf_op_rate_total")
+            run.perf_avg_latency_99th = performance_results.get("perf_avg_latency_99th")
+            run.perf_avg_latency_mean = performance_results.get("perf_avg_latency_mean")
+            run.perf_total_errors = performance_results.get("perf_total_errors")
+            run.stress_cmd = performance_results.get("stress_cmd")
+            run.test_name = performance_results.get("test_name")
+            run.save()
+
+            is_latency_test = "latency" in run.test_name
+            threshold_negative = -10
+
+            def cmp(lhs, rhs):
+                delta = rhs - lhs
+                change = int(math.fabs(delta) * 100 / rhs)
+                return change if delta >= 0 else change * -1
+
+            previous_runs = SCTTestRun.get_perf_results_for_test_name(run.build_id, run.start_time, run.test_name)
+            metrics_to_check = ["perf_avg_latency_99th",
+                                "perf_avg_latency_mean"] if is_latency_test else ["perf_op_rate_total"]
+
+            older_runs_by_version = {}
+            for prev_run in previous_runs:
+                if not older_runs_by_version.get(prev_run["scylla_version"]):
+                    older_runs_by_version[prev_run["scylla_version"]] = []
+                older_runs_by_version[prev_run["scylla_version"]].append(prev_run)
+
+            regression_found = False
+            regression_info = {
+                "version": None,
+                "delta": None,
+                "id": None,
+                "metric": None,
+                "job_url": None,
+            }
+
+            if performance_results["histograms"]:
+                for histogram in performance_results["histograms"]:
+                    run.histograms = {k: PerformanceHDRHistogram(**v) for k, v in histogram.items()}
+
+            for version, runs in older_runs_by_version.items():
+                for metric in metrics_to_check:
+                    best_run = sorted(runs, reverse=(not is_latency_test), key=lambda v: v[metric])[0]
+                    last_run = runs[0]
+
+                    metric_to_best = cmp(run[metric], best_run[metric])
+                    metric_to_last = cmp(run[metric], last_run[metric])
+                    if metric_to_last < threshold_negative:
+                        regression_found = True
+                        regression_info["metric"] = metric
+                        regression_info["version"] = version
+                        regression_info["job_url"] = last_run["build_job_url"]
+                        regression_info["id"] = str(last_run["id"])
+                        regression_info["delta"] = metric_to_last
+                        break
+
+                    if metric_to_best < threshold_negative:
+                        regression_found = True
+                        regression_info["metric"] = metric
+                        regression_info["version"] = version
+                        regression_info["job_url"] = best_run["build_job_url"]
+                        regression_info["id"] = str(best_run["id"])
+                        regression_info["delta"] = metric_to_best
+                        break
+
+                if regression_found:
+                    break
+
+            if regression_found:
+                run.status = TestStatus.FAILED.value
+                run.save()
+                EventService.create_run_event(kind=ArgusEventTypes.TestRunStatusChanged, body={
+                    "message": "[{username}] Setting run status to {status} due to performance metric '{metric}' falling "
+                    "below allowed threshold ({threshold_negative}): {delta}% compared to "
+                    "<a href='/test/{test_id}/runs?additionalRuns[]={base_run_id}&additionalRuns[]={previous_run_id}'>This {version} (#{build_number}) run</a>",
+                    "username": g.user.username,
+                    "status": TestStatus.FAILED.value,
+                    "metric": regression_info["metric"],
+                    "threshold_negative": threshold_negative,
+                    "delta": regression_info["delta"],
+                    "test_id": str(run.test_id),
+                    "base_run_id": str(run.id),
+                    "previous_run_id": regression_info["id"],
+                    "version": regression_info["version"],
+                    "build_number": get_build_number(regression_info["job_url"])
+                }, user_id=g.user.id, run_id=run_id, release_id=run.release_id, test_id=run.test_id)
+            else:
+                # NOTE: This will override status set by SCT Events.
+                run.status = TestStatus.PASSED.value
+                run.save()
+
+        except SCTTestRun.DoesNotExist as exception:
+            LOGGER.error("Run %s not found for SCTTestRun", run_id)
+            raise SCTServiceException("Run not found", run_id) from exception
+
+        return "submitted"
+
+    @staticmethod
+    def get_performance_history_for_test(run_id: str):
+        try:
+            run: SCTTestRun = SCTTestRun.get(id=run_id)
+            rows = run.get_perf_results_for_test_name(
+                build_id=run.build_id, start_time=run.start_time, test_name=run.test_name)
+            return rows
+        except SCTTestRun.DoesNotExist as exception:
+            LOGGER.error("Run %s not found for SCTTestRun", run_id)
+            raise SCTServiceException("Run not found", run_id) from exception
+
+    @staticmethod
+    def create_resource(run_id: str, resource_details: dict) -> str:
+        instance_details = CloudInstanceDetails(**resource_details.pop("instance_details"))
+        resource = CloudResource(**resource_details, instance_info=instance_details)
+        try:
+            run: SCTTestRun = SCTTestRun.get(id=run_id)
+            run.get_resources().append(resource)
+            run.save()
+        except SCTTestRun.DoesNotExist as exception:
+            LOGGER.error("Run %s not found for SCTTestRun", run_id)
+            raise SCTServiceException("Run not found", run_id) from exception
+
+        return "created"
+
+    @staticmethod
+    def update_resource_shards(run_id: str, resource_name: str, new_shards: int) -> str:
+        try:
+            run: SCTTestRun = SCTTestRun.get(id=run_id)
+            resource = next(res for res in run.get_resources() if res.name == resource_name)
+            resource.get_instance_info().shards_amount = new_shards
+            run.save()
+        except StopIteration as exception:
+            LOGGER.error("Resource %s not found in run %s", resource_name, run_id)
+            raise SCTServiceException("Resource not found", resource_name) from exception
+        except SCTTestRun.DoesNotExist as exception:
+            LOGGER.error("Run %s not found for SCTTestRun", run_id)
+            raise SCTServiceException("Run not found", run_id) from exception
+
+        return "updated"
+
+    @staticmethod
+    def update_resource(run_id: str, resource_name: str, update_data: ResourceUpdateRequest) -> str:
+        try:
+            fields_updated = {}
+            run: SCTTestRun = SCTTestRun.get(id=run_id)
+            resource = next(res for res in run.get_resources() if res.name == resource_name)
+            instance_info = update_data.pop("instance_info", None)
+            resource.state = ResourceState(update_data.get("state", resource.state)).value
+            if instance_info:
+                resource_instance_info = resource.get_instance_info()
+                for k, v in instance_info.items():
+                    if k in resource_instance_info.keys():
+                        resource_instance_info[k] = v
+                        fields_updated[k] = v
+            run.save()
+        except StopIteration as exception:
+            LOGGER.error("Resource %s not found in run %s", resource_name, run_id)
+            raise SCTServiceException("Resource not found", resource_name) from exception
+        except SCTTestRun.DoesNotExist as exception:
+            LOGGER.error("Run %s not found for SCTTestRun", run_id)
+            raise SCTServiceException("Run not found", run_id) from exception
+
+        return {
+            "state": "updated",
+            "fields": fields_updated
+        }
+
+    @staticmethod
+    def terminate_resource(run_id: str, resource_name: str, reason: str) -> str:
+        try:
+            run: SCTTestRun = SCTTestRun.get(id=run_id)
+            if "sct-runner" in resource_name:  # FIXME: Temp solution until sct-runner name is propagated on submit
+                resource = next(res for res in run.get_resources() if "sct-runner" in res.name)
+            else:
+                resource = next(res for res in run.get_resources() if res.name == resource_name)
+            resource.get_instance_info().termination_reason = reason
+            resource.get_instance_info().termination_time = int(time())
+            resource.state = ResourceState.TERMINATED.value
+            run.save()
+        except StopIteration as exception:
+            LOGGER.error("Resource %s not found in run %s", resource_name, run_id)
+            raise SCTServiceException("Resource not found", resource_name) from exception
+        except SCTTestRun.DoesNotExist as exception:
+            LOGGER.error("Run %s not found for SCTTestRun", run_id)
+            raise SCTServiceException("Run not found", run_id) from exception
+
+        return "terminated"
+
+    @staticmethod
+    def submit_nemesis(run_id: str, nemesis_details: dict) -> str:
+        nem_req = NemesisSubmissionRequest(**nemesis_details)
+        node_desc = NodeDescription(name=nem_req.node_name, ip=nem_req.node_ip, shards=nem_req.node_shards)
+        nemesis_info = NemesisRunInfo(
+            class_name=nem_req.class_name,
+            name=nem_req.name,
+            start_time=int(nem_req.start_time),
+            end_time=0,
+            duration=0,
+            stack_trace="",
+            status=NemesisStatus.RUNNING.value,
+            target_node=node_desc,
+        )
+        try:
+            run: SCTTestRun = SCTTestRun.get(id=run_id)
+            run.add_nemesis(nemesis_info)
+            run.save()
+        except SCTTestRun.DoesNotExist as exception:
+            LOGGER.error("Run %s not found for SCTTestRun", run_id)
+            raise SCTServiceException("Run not found", run_id) from exception
+
+        return "created"
+
+    @staticmethod
+    def finalize_nemesis(run_id: str, nemesis_details: dict) -> str:
+        nem_req = NemesisFinalizationRequest(**nemesis_details)
+        try:
+            run: SCTTestRun = SCTTestRun.get(id=run_id)
+            nemesis = next(nem for nem in run.get_nemeses() if nem.name ==
+                           nem_req.name and nem.start_time == nem_req.start_time)
+            nemesis.status = NemesisStatus(nem_req.status).value
+            nemesis.stack_trace = nem_req.message
+            nemesis.end_time = int(time())
+            run.save()
+        except StopIteration as exception:
+            LOGGER.error("Nemesis %s (%s) not found for run %s", nem_req.name, nem_req.start_time, run_id)
+            raise SCTServiceException("Nemesis not found", (nem_req.name, nem_req.start_time)) from exception
+        except SCTTestRun.DoesNotExist as exception:
+            LOGGER.error("Run %s not found for SCTTestRun", run_id)
+            raise SCTServiceException("Run not found", run_id) from exception
+
+        return "updated"
+
+    @staticmethod
+    def submit_events(run_id: str, events: list[dict]) -> str:
+        wrapped_events = [EventSubmissionRequest(**ev) for ev in events]
+        try:
+            run: SCTTestRun = SCTTestRun.get(id=run_id)
+            for event in wrapped_events:
+                wrapper = EventsBySeverity(severity=event.severity,
+                                           event_amount=event.total_events, last_events=event.messages)
+                run.get_events().append(wrapper)
+            coredumps = SCTService.locate_coredumps(run, run.get_events())
+            run.submit_logs(coredumps)
+            run.save()
+        except SCTTestRun.DoesNotExist as exception:
+            LOGGER.error("Run %s not found for SCTTestRun", run_id)
+            raise SCTServiceException("Run not found", run_id) from exception
+
+        return "added"
+
+    @staticmethod
+    def locate_coredumps(run: SCTTestRun, events: list[EventsBySeverity]) -> list[dict]:
+        flat_messages: list[str] = []
+        links = []
+        for es in events:
+            flat_messages.extend(es.last_events)
+        coredump_events = filter(lambda v: "coredumpevent" in v.lower(), flat_messages)
+        for idx, event in enumerate(coredump_events):
+            core_pattern = r"corefile_url=(?P<url>.+)$"
+            ts_pattern = r"^(?P<ts>\d{4}-\d{2}-\d{2} ([\d:]*)\.\d{3})"
+            node_name_pattern = r"node=(?P<name>.+)$"
+            core_url_match = re.search(core_pattern, event, re.MULTILINE)
+            node_name_match = re.search(node_name_pattern, event, re.MULTILINE)
+            ts_match = re.search(ts_pattern, event)
+            if core_url_match:
+                node_name = node_name_match.group("name") if node_name_match else f"unknown-node-{idx}"
+                split_name = node_name.split(" ")
+                node_name = split_name[1] if len(split_name) >= 2 else node_name
+                url = core_url_match.group("url")
+                timestamp_component = ""
+                if ts_match:
+                    try:
+                        timestamp = datetime.fromisoformat(ts_match.group("ts"))
+                        timestamp_component = timestamp.strftime("-%Y-%m-%d_%H-%M-%S")
+                    except ValueError:
+                        pass
+                log_link = {
+                    "log_name": f"core.scylla-{node_name}{timestamp_component}.gz",
+                    "log_link": url
+                }
+                links.append(log_link)
+        return links
+
+    @staticmethod
+    def get_similar_events(run_id: str) -> list[dict]:
+        """Get similar events for each event in a test run
+
+        Args:
+            run_id: The test run ID to get similar events for
+
+        Returns:
+            List of dictionaries containing event_index, severity and similars_set for each event
+        """
+        error_embeddings = ErrorEventEmbeddings.filter(run_id=run_id).only(["event_index", "similars_map"]).all()
+        critical_embeddings = CriticalEventEmbeddings.filter(run_id=run_id).only(["event_index", "similars_map"]).all()
+
+        result = []
+        # Process ERROR embeddings
+        for embedding in error_embeddings:
+            result.append(
+                {
+                    "event_index": embedding.event_index,
+                    "severity": "ERROR",
+                    "similars_set": [str(similar_run_id) for similar_run_id in embedding.similars_map],
+                }
+            )
+
+        # Process CRITICAL embeddings
+        for embedding in critical_embeddings:
+            result.append(
+                {
+                    "event_index": embedding.event_index,
+                    "severity": "CRITICAL",
+                    "similars_set": [str(similar_run_id) for similar_run_id in embedding.similars_map],
+                }
+            )
+
+        return result
+
+    @staticmethod
+    def get_similar_runs_info(run_ids: list[str]):
+        """Get build IDs and issues for a list of run IDs
+
+        Args:
+            run_ids: List of run IDs to fetch information for
+
+        Returns:
+            Dictionary mapping run IDs to their information (build_id, start_time, and issues)
+        """
+        result = {}
+
+        # Step 1: Get issue links for all run_ids in batches
+        all_issue_links = {}
+
+        for batch_run_ids in chunk(run_ids):
+            batch_links = IssueLink.objects.filter(run_id__in=batch_run_ids).all()
+
+            for link in batch_links:
+                run_id_str = str(link.run_id)
+                if run_id_str not in all_issue_links:
+                    all_issue_links[run_id_str] = []
+                all_issue_links[run_id_str].append(link)
+
+        # Step 2: Fetch all unique issue details
+        all_issue_ids = set()
+        for links in all_issue_links.values():
+            all_issue_ids.update(link.issue_id for link in links)
+
+        issues_by_id = {}
+        if all_issue_ids:
+            for batch_issue_ids in chunk(list(all_issue_ids)):
+                batch_issues = GithubIssue.filter(id__in=batch_issue_ids).all()
+
+                for issue in batch_issues:
+                    issues_by_id[issue.id] = issue
+
+        # Step 3: Fetch test runs only for run_ids that have issue links (limiting to MAX_SIMILARS runs)
+        runs_with_issues = list(all_issue_links.keys())
+
+        test_runs = {}
+        if runs_with_issues:
+            for run_id in runs_with_issues[:MAX_SIMILARS]:
+                try:
+                    test_run = SCTTestRun.get(id=run_id)
+                    test_runs[run_id] = test_run
+                except Exception as e:
+                    LOGGER.debug(f"Failed to fetch test run {run_id}: {str(e)}")
+
+        # Step 4: Assign run and issue details to result for runs with issues
+        for run_id in runs_with_issues:
+            try:
+                test_run = test_runs.get(run_id)
+                if not test_run:
+                    continue
+
+                links = all_issue_links.get(run_id, [])
+                issues = [issues_by_id[link.issue_id] for link in links if link.issue_id in issues_by_id]
+
+                try:
+                    build_number = int(
+                        test_run.build_job_url[:-1].split("/")[-1]
+                    )
+                except Exception as e:
+                    LOGGER.error(
+                        f"Error parsing build number for run {run_id}: {test_run.build_job_url[:-1].split('/')} - {str(e)}")
+                    build_number = -1
+
+                for pkg_name in ["scylla-server-upgraded", "scylla-server", "scylla-server-target"]:
+                    sut_version = next(
+                        (f"{pkg.version}-{pkg.date}" for pkg in test_run.packages if pkg.name == pkg_name), None)
+                    if sut_version:
+                        break
+
+                result[run_id] = {
+                    "build_id": f"{test_run.build_id}#{build_number}",
+                    "start_time": test_run.start_time.isoformat(),
+                    "version": sut_version or "unknown",
+                    "issues": [
+                        {
+                            "number": issue.number,
+                            "state": issue.state,
+                            "title": issue.title,
+                            "url": issue.url,
+                        }
+                        for issue in issues
+                    ],
+                }
+            except Exception as e:
+                LOGGER.error(f"Error fetching info for run {run_id}: {str(e)}")
+                result[run_id] = {"build_id": None, "start_time": None, "issues": []}
+
+        # Step 5: If less than MAX_SIMILARS results, fetch MAX_SIMILARS more run details
+        if len(result) < MAX_SIMILARS:
+            remaining_run_ids = [run_id for run_id in run_ids if run_id not in result]
+            additional_needed = min(MAX_SIMILARS - len(result), len(remaining_run_ids))
+
+            if additional_needed > 0:
+                additional_run_ids = remaining_run_ids[:additional_needed]
+
+                additional_test_runs = {}
+                for run_id in additional_run_ids:
+                    try:
+                        test_run = SCTTestRun.get(id=run_id)
+                        additional_test_runs[run_id] = test_run
+                    except Exception as e:
+                        LOGGER.debug(f"Failed to fetch additional test run {run_id}: {str(e)}")
+
+                for run_id in additional_run_ids:
+                    try:
+                        test_run = additional_test_runs.get(run_id)
+                        if not test_run:
+                            continue
+
+                        try:
+                            build_number = int(
+                                test_run.build_job_url[:-1].split("/")[-1]
+                            )
+                        except Exception as e:
+                            LOGGER.error(
+                                f"Error parsing build number for run {run_id}: {test_run.build_job_url[:-1].split('/')} - {str(e)}")
+                            build_number = -1
+
+                        for pkg_name in ["scylla-server-upgraded", "scylla-server", "scylla-server-target"]:
+                            sut_version = next(
+                                (f"{pkg.version}-{pkg.date}" for pkg in test_run.packages if pkg.name == pkg_name), None)
+                            if sut_version:
+                                break
+
+                        result[run_id] = {
+                            "build_id": f"{test_run.build_id}#{build_number}",
+                            "start_time": test_run.start_time.isoformat(),
+                            "version": sut_version or "unknown",
+                            "issues": [],
+                        }
+                    except Exception as e:
+                        LOGGER.error(f"Error fetching info for run {run_id}: {str(e)}")
+                        result[run_id] = {"build_id": None, "start_time": None, "issues": []}
+
+        return result
+
+    @staticmethod
+    def get_scylla_version_kernels_report(release_name: str):
+        all_release_runs = SCTTestRun.get_version_data_for_release(release_name=release_name)
+        kernels_by_version = {}
+        kernel_metadata = {}
+        for run in all_release_runs:
+            packages = run["packages"]
+            if not packages:
+                continue
+            scylla_pkgs = {p["name"]: p for p in packages if "scylla-server" in p["name"]}
+            scylla_pkg = scylla_pkgs["scylla-server-upgraded"] if scylla_pkgs.get(
+                "scylla-server-upgraded") else scylla_pkgs.get("scylla-server")
+            version = f"{scylla_pkg['version']}-{scylla_pkg['date']}.{scylla_pkg['revision_id']}" if scylla_pkgs else "unknown"
+            kernel_packages = [p for p in packages if "kernel" in p["name"]]
+            kernel_package = kernel_packages[0] if len(kernel_packages) > 0 else None
+            if not kernel_package:
+                continue
+            version_list = set(kernels_by_version.get(version, []))
+            version_list.add(kernel_package["version"])
+            kernels_by_version[version] = list(version_list)
+            metadata = kernel_metadata.get(
+                kernel_package.version,
+                {
+                    "passed": 0,
+                    "failed": 0,
+                    "aborted": 0,
+                }
+            )
+            if run["status"] in ["passed", "failed", "aborted", "test_error"]:
+                metadata[run["status"]] += 1
+            kernel_metadata[kernel_package["version"]] = metadata
+
+        return {
+            "versions": kernels_by_version,
+            "metadata": kernel_metadata
+        }
+
+    @staticmethod
+    def junit_submit(run_id: str, file_name: str, content: str) -> bool:
+        try:
+            report = SCTJunitReports.get(test_id=run_id, file_name=file_name)
+            if report:
+                raise SCTServiceException(f"Report {file_name} already exists.", file_name)
+        except SCTJunitReports.DoesNotExist:
+            pass
+        report = SCTJunitReports()
+        report.test_id = run_id
+        report.file_name = file_name
+
+        xml_content = str(base64.decodebytes(bytes(content, encoding="utf-8")), encoding="utf-8")
+        try:
+            _ = ElementTree.fromstring(xml_content)
+        except Exception:
+            raise SCTServiceException(f"Malformed JUnit report submitted")
+
+        report.report = xml_content
+        report.save()
+
+        return True
+
+    @staticmethod
+    def junit_get_all(run_id: str) -> list[SCTJunitReports]:
+        return list(SCTJunitReports.filter(test_id=run_id).all())
+
+    @staticmethod
+    def junit_get_single(run_id: str, file_name: str) -> SCTJunitReports:
+        return SCTJunitReports.get(test_id=run_id, file_name=file_name)