argus-alm 0.12.9__py3-none-any.whl → 0.13.0__py3-none-any.whl

argus/backend/service/build_system_monitor.py

@@ -1,188 +0,0 @@
-import logging
-from abc import ABC, abstractmethod
-import jenkins
-import click
-import re
-from flask import current_app
-from flask.cli import with_appcontext
-
-from argus.backend.db import ScyllaCluster
-from argus.backend.models.web import ArgusRelease, ArgusGroup, ArgusTest, ArgusTestException
-from argus.backend.service.release_manager import ReleaseManagerService
-
-LOGGER = logging.getLogger(__name__)
-
-
-class ArgusTestsMonitor(ABC):
-    BUILD_SYSTEM_FILTERED_PREFIXES = [
-
-    ]
-
-    def __init__(self) -> None:
-        self._cluster = ScyllaCluster.get()
-        self._existing_releases = list(ArgusRelease.objects().limit(None))
-        self._existing_groups = list(ArgusGroup.objects().limit(None))
-        self._existing_tests = list(ArgusTest.objects().limit(None))
-        self._filtered_groups: list[str] = self.BUILD_SYSTEM_FILTERED_PREFIXES
-
-    def create_release(self, release_name):
-        # pylint: disable=no-self-use
-        release = ArgusRelease()
-        release.name = release_name
-        release.save()
-
-        return release
-
-    def create_group(self, release: ArgusRelease, group_name: str, build_id: str, group_pretty_name: str | None = None):
-        # pylint: disable=no-self-use
-        group = ArgusGroup()
-        group.release_id = release.id
-        group.name = group_name
-        group.build_system_id = build_id
-        if group_pretty_name:
-            group.pretty_name = group_pretty_name
-        group.save()
-
-        return group
-
-    def create_test(self, release: ArgusRelease, group: ArgusGroup,
-                    test_name: str, build_id: str, build_url: str) -> ArgusTest:
-        # pylint: disable=no-self-use
-        test = ArgusTest()
-        test.name = test_name
-        test.group_id = group.id
-        test.release_id = release.id
-        test.build_system_id = build_id
-        test.build_system_url = build_url
-        test.validate_build_system_id()
-        test.save()
-        ReleaseManagerService().move_test_runs(test)
-
-        return test
-
-    @abstractmethod
-    def collect(self):
-        raise NotImplementedError()
-
-    def check_filter(self, group_name: str) -> bool:
-        for prefix in self._filtered_groups:
-            if prefix.lower() in group_name.lower():
-                return False
-
-        return True
-
-
-class JenkinsMonitor(ArgusTestsMonitor):
-
-    BUILD_SYSTEM_FILTERED_PREFIXES = [
-        "releng",
-    ]
-
-    JENKINS_MONITORED_RELEASES = [
-        r"^scylla-master$",
-        r"^scylla-staging$",
-        r"^scylla-\d+\.\d+$",
-        r"^manager-3.\d+$",
-        r"^scylla-operator/operator-master$",
-        r"^scylla-operator/operator-\d+.\d+$",
-        r"^scylla-enterprise$",
-        r"^enterprise-20\d{2}\.\d+$",
-        r"^siren-tests$",
-    ]
-
-    def __init__(self) -> None:
-        super().__init__()
-        self._jenkins = jenkins.Jenkins(url=current_app.config["JENKINS_URL"],
-                                        username=current_app.config["JENKINS_USER"],
-                                        password=current_app.config["JENKINS_API_TOKEN"])
-        self._monitored_releases = self.JENKINS_MONITORED_RELEASES
-
-    def _check_release_name(self, release_name: str):
-        return any(re.match(pattern, release_name, re.IGNORECASE) for pattern in self._monitored_releases)
-
-    def collect(self):
-        click.echo("Collecting new tests from jenkins")
-        all_jobs = self._jenkins.get_all_jobs()
-        all_monitored_folders = [job for job in all_jobs if self._check_release_name(job["fullname"])]
-        LOGGER.info("Will collect %s", [f["fullname"] for f in all_monitored_folders])
-
-        for release in all_monitored_folders:
-            LOGGER.info("Processing release %s", release["name"])
-            try:
-                saved_release = ArgusRelease.get(name=release["name"])
-                LOGGER.info("Release %s exists", release["name"])
-            except ArgusRelease.DoesNotExist:
-                LOGGER.warning("Release %s does not exist, creating...", release["name"])
-                saved_release = self.create_release(release["name"])
-                self._existing_releases.append(saved_release)
-
-            try:
-                groups = self.collect_groups_for_release(release["jobs"])
-            except KeyError:
-                LOGGER.error("Empty release!\n %s", release)
-                continue
-            folder_stack = [dict(parent_name="", parent_display_name="", group=g) for g in reversed(groups)]
-            root_folder = {
-                "parent_name": "",
-                "parent_display_name": "",
-                "group":  {
-                    "name": f"{release['fullname']}-root",
-                    "displayName": "-- root directory --",
-                    "fullname": release["fullname"],
-                    "jobs": self.collect_root_folder_jobs(release["jobs"]),
-                }
-            }
-            folder_stack.append(root_folder)
-            while len(folder_stack) != 0:
-                group_dict = folder_stack.pop()
-                group = group_dict["group"]
-                LOGGER.info("Processing group %s for release %s", group["name"], saved_release.name)
-                try:
-                    group_name = group["name"] if not group_dict["parent_name"] else f"{group_dict['parent_name']}-{group['name']}"
-                    saved_group = filter(lambda g: g.build_system_id == group["fullname"], self._existing_groups)
-                    saved_group = next(saved_group)
-                    LOGGER.info("Group %s already exists. (id: %s)", saved_group.build_system_id, saved_group.id)
-                except StopIteration:
-                    LOGGER.warning(
-                        "Group %s for release %s doesn't exist, creating...", group_name, saved_release.name)
-                    try:
-                        display_name = group.get("displayName", self._jenkins.get_job_info(name=group["fullname"])["displayName"])
-                        display_name = display_name if not group_dict[
-                            "parent_display_name"] else f"{group_dict['parent_display_name']} - {display_name}"
-                    except Exception:
-                        display_name = None
-
-                    saved_group = self.create_group(saved_release, group_name, group["fullname"], display_name)
-                    self._existing_groups.append(saved_group)
-
-                for job in group["jobs"]:
-                    LOGGER.info("Processing job %s for release %s and group %s",
-                                job["fullname"], saved_group.name, saved_release.name)
-                    saved_test = None
-                    if "Folder" in job["_class"]:
-                        folder_stack.append(dict(parent_name=saved_group.name,
-                                            parent_display_name=saved_group.pretty_name, group=job))
-                    if "WorkflowJob" in job["_class"]:
-                        try:
-                            saved_test = filter(lambda t: t.build_system_id == job["fullname"], self._existing_tests)
-                            saved_test = next(saved_test)
-                            LOGGER.info("Test %s already exists. (id: %s)", saved_test.build_system_id, saved_test.id)
-                        except StopIteration:
-                            LOGGER.warning("Test %s for release %s (group %s) doesn't exist, creating...",
-                                           job["name"], saved_release.name, saved_group.name)
-                            try:
-                                saved_test = self.create_test(
-                                    saved_release, saved_group, job["name"], job["fullname"], job["url"])
-                                self._existing_tests.append(saved_test)
-                            except ArgusTestException:
-                                LOGGER.error("Unable to create test for build_id %s", job["fullname"], exc_info=True)
-
-    def collect_groups_for_release(self, jobs):
-        # pylint: disable=no-self-use
-        groups = [folder for folder in jobs if "Folder" in folder["_class"]]
-        groups = [group for group in groups if self.check_filter(group["name"])]
-
-        return groups
-
-    def collect_root_folder_jobs(self, jobs):
-        return [job for job in jobs if "WorkflowJob" in job["_class"]]

argus/backend/service/client_service.py

@@ -1,122 +0,0 @@
-import operator
-from dataclasses import asdict, is_dataclass
-from datetime import datetime, timezone
-from functools import partial
-from uuid import UUID
-
-from argus.backend.db import ScyllaCluster
-from argus.backend.models.result import ArgusGenericResultMetadata, ArgusGenericResultData
-from argus.backend.plugins.core import PluginModelBase
-from argus.backend.plugins.loader import AVAILABLE_PLUGINS
-from argus.backend.service.results_service import ResultsService, Cell
-from argus.backend.util.enums import TestStatus
-
-
-class ClientException(Exception):
-    pass
-
-
-class ClientService:
-    PLUGINS = {name: plugin.model for name, plugin in AVAILABLE_PLUGINS.items()}
-
-    def __init__(self) -> None:
-        self.cluster = ScyllaCluster.get()
-
-    def get_model(self, run_type: str) -> PluginModelBase:
-        cls = self.PLUGINS.get(run_type)
-        if not cls:
-            raise ClientException(f"Unsupported run type: {run_type}", run_type)
-        return cls
-
-    def submit_run(self, run_type: str, request_data: dict) -> str:
-        model = self.get_model(run_type)
-        model.submit_run(request_data=request_data)
-
-        return "Created"
-
-    def get_run(self, run_type: str, run_id: str):
-        model = self.get_model(run_type)
-        try:
-            run = model.get(id=run_id)
-        except model.DoesNotExist:
-            return None
-        return run
-
-    def heartbeat(self, run_type: str, run_id: str) -> int:
-        model = self.get_model(run_type)
-        run = model.load_test_run(UUID(run_id))
-        run.update_heartbeat()
-        run.save()
-        return run.heartbeat
-
-    def get_run_status(self, run_type: str, run_id: str) -> str:
-        model = self.get_model(run_type)
-        run = model.load_test_run(UUID(run_id))
-        return run.status
-
-    def update_run_status(self, run_type: str, run_id: str, new_status: str) -> str:
-        model = self.get_model(run_type)
-        run = model.load_test_run(UUID(run_id))
-        run.change_status(new_status=TestStatus(new_status))
-        run.save()
-
-        return run.status
-
-    def submit_product_version(self, run_type: str, run_id: str, version: str) -> str:
-        model = self.get_model(run_type)
-        run = model.load_test_run(UUID(run_id))
-        run.submit_product_version(version)
-        run.save()
-
-        return "Submitted"
-
-    def submit_logs(self, run_type: str, run_id: str, logs: list[dict]) -> str:
-        model = self.get_model(run_type)
-        run = model.load_test_run(UUID(run_id))
-        run.submit_logs(logs)
-        run.save()
-
-        return "Submitted"
-
-    def finish_run(self, run_type: str, run_id: str, payload: dict | None = None) -> str:
-        model = self.get_model(run_type)
-        run = model.load_test_run(UUID(run_id))
-        run.finish_run(payload)
-        run.save()
-
-        return "Finalized"
-
-    def submit_results(self, run_type: str, run_id: str, results: dict) -> dict[str, str]:
-        model = self.get_model(run_type)
-        try:
-            run = model.load_test_run(UUID(run_id))
-        except model.DoesNotExist:
-            return {"status": "error", "response": {
-                "exception": "DoesNotExist",
-                "arguments": [run_id]
-            }}
-        table_name = results["meta"]["name"]
-        results_service = ResultsService()
-        cells = [Cell(**cell) for cell in results["results"]]
-        table_metadata = results_service.get_table_metadata(test_id=run.test_id, table_name=table_name)
-        if table_metadata:
-            table_metadata = table_metadata.update_if_changed(results["meta"])
-        else:
-            table_metadata = ArgusGenericResultMetadata(test_id=run.test_id, **results["meta"])
-            table_metadata.save()
-        if results.get("sut_timestamp", 0) == 0:
-            results["sut_timestamp"] = run.sut_timestamp()  # automatic sut_timestamp
-        results["sut_timestamp"] = datetime.fromtimestamp(results["sut_timestamp"])
-        best_results = results_service.update_best_results(test_id=run.test_id, table_name=table_name, table_metadata=table_metadata,
-                                                           cells=cells, run_id=run_id)
-        table_name = results["meta"]["name"]
-        sut_timestamp = results["sut_timestamp"]
-        for cell in cells:
-            cell.update_cell_status_based_on_rules(table_metadata, best_results)
-            ArgusGenericResultData(test_id=run.test_id,
-                                   run_id=run.id,
-                                   name=table_name,
-                                   sut_timestamp=sut_timestamp,
-                                   **asdict(cell)
-                                   ).save()
-        return {"status": "ok", "message": "Results submitted"}

argus/backend/service/event_service.py

@@ -1,18 +0,0 @@
-from datetime import datetime
-import json
-from argus.backend.models.web import ArgusEvent, ArgusEventTypes
-
-
-class EventService:
-    @staticmethod
-    def create_run_event(kind: ArgusEventTypes, body: dict, user_id=None, run_id=None, release_id=None, group_id=None, test_id=None):
-        event = ArgusEvent()
-        event.release_id = release_id
-        event.group_id = group_id
-        event.test_id = test_id
-        event.user_id = user_id
-        event.run_id = run_id
-        event.body = json.dumps(body, ensure_ascii=True, separators=(',', ':'))
-        event.kind = kind.value
-        event.created_at = datetime.utcnow()
-        event.save()

argus/backend/service/jenkins_service.py

@@ -1,240 +0,0 @@
-import re
-import requests
-from typing import Any, TypedDict
-from uuid import UUID
-import xml.etree.ElementTree as ET
-import jenkins
-import logging
-
-from flask import current_app, g
-
-from argus.backend.models.web import ArgusGroup, ArgusRelease, ArgusTest, UserOauthToken
-
-LOGGER = logging.getLogger(__name__)
-GITHUB_REPO_RE = r"(?P<http>^https?:\/\/(www\.)?github\.com\/(?P<user>[\w\d\-]+)\/(?P<repo>[\w\d\-]+)(\.git)?$)|(?P<ssh>git@github\.com:(?P<ssh_user>[\w\d\-]+)\/(?P<ssh_repo>[\w\d\-]+)(\.git)?)"
-
-class Parameter(TypedDict):
-    _class: str
-    name: str
-    description: str
-    value: Any
-
-
-class JenkinsServiceError(Exception):
-    pass
-
-
-class JenkinsService:
-    RESERVED_PARAMETER_NAME = "requested_by_user"
-
-    SETTINGS_CONFIG_MAP = {
-        "scylla-cluster-tests": {
-            "gitRepo": "*//scm/userRemoteConfigs/hudson.plugins.git.UserRemoteConfig/url",
-            "gitBranch": "*//scm/branches/hudson.plugins.git.BranchSpec/name",
-            "pipelineFile": "*//scriptPath",
-        },
-        "driver-matrix-tests": {
-            "gitRepo": "*//scm/userRemoteConfigs/hudson.plugins.git.UserRemoteConfig/url",
-            "gitBranch": "*//scm/branches/hudson.plugins.git.BranchSpec/name",
-            "pipelineFile": "*//scriptPath",
-        },
-        "sirenada": {
-            "gitRepo": "*//scm/userRemoteConfigs/hudson.plugins.git.UserRemoteConfig/url",
-            "gitBranch": "*//scm/branches/hudson.plugins.git.BranchSpec/name",
-            "pipelineFile": "*//scriptPath",
-        }
-    }
-
-    def __init__(self) -> None:
-        self._jenkins = jenkins.Jenkins(url=current_app.config["JENKINS_URL"],
-                                        username=current_app.config["JENKINS_USER"],
-                                        password=current_app.config["JENKINS_API_TOKEN"])
-
-    def retrieve_job_parameters(self, build_id: str, build_number: int) -> list[Parameter]:
-        job_info = self._jenkins.get_build_info(name=build_id, number=build_number)
-        raw_config = self._jenkins.get_job_config(name=build_id)
-        config = ET.fromstring(raw_config)
-        parameter_defs = config.find("*//parameterDefinitions")
-        if parameter_defs:
-            descriptions = {
-                define.findtext("name"): f"{define.findtext('description', '')}" + f" (default: <span class=\"fw-bold\">{define.findtext('defaultValue')}</span>)" if define.findtext('defaultValue') else ""
-                for define in parameter_defs.iterfind("hudson.model.StringParameterDefinition")
-            }
-        else:
-            descriptions = {}
-        params = next(a for a in job_info["actions"] if a.get("_class", "#NONE") == "hudson.model.ParametersAction")["parameters"]
-        params = [param for param in params if param["name"] != self.RESERVED_PARAMETER_NAME]
-        for idx, param in enumerate(params):
-            params[idx]["description"] = descriptions.get(param["name"], "")
-
-        return params
-
-    def get_releases_for_clone(self, test_id: str):
-        test_id = UUID(test_id)
-        # TODO: Filtering based on origin location / user preferences
-        _: ArgusTest = ArgusTest.get(id=test_id)
-
-        releases = list(ArgusRelease.all())
-
-        return sorted(releases, key=lambda r: r.pretty_name if r.pretty_name else r.name)
-
-    def get_groups_for_release(self, release_id: str):
-        groups = list(ArgusGroup.filter(release_id=release_id).all())
-
-        return sorted(groups, key=lambda g: g.pretty_name if g.pretty_name else g.name)
-
-    def _verify_sct_settings(self, new_settings: dict[str, str]) -> tuple[bool, str]:
-        if not (match := re.match(GITHUB_REPO_RE, new_settings["gitRepo"])):
-            return (False, "Repository doesn't conform to GitHub schema")
-
-        git_info = match.groupdict()
-        if git_info.get("ssh"):
-            repo = git_info["ssh_repo"]
-            user = git_info["ssh_user"]
-        else:
-            repo = git_info["repo"]
-            user = git_info["user"]
-
-        user_tokens = UserOauthToken.filter(user_id=g.user.id).all()
-        token = None
-        for tok in user_tokens:
-            if tok.kind == "github":
-                token = tok.token
-                break
-        if not token:
-            raise JenkinsServiceError("Github token not found")
-
-        response = requests.get(
-            url=f"https://api.github.com/repos/{user}/{repo}/contents/{new_settings['pipelineFile']}?ref={new_settings['gitBranch']}",
-            headers={
-                "Accept": "application/vnd.github+json",
-                "Authorization": f"Bearer {token}",
-            }
-            )
-        
-        if response.status_code == 404:
-            return (False, f"Pipeline file not found in the <a href=\"https://github.com/{user}/{repo}/tree/{new_settings['gitBranch']}\"> target repository</a>, please check the repository before continuing")
-
-        if response.status_code == 403:
-            return (True, "No access to this repository using your token. The pipeline file cannot be verified.")
-
-        if response.status_code == 200:
-            return (True, "")
-
-        return (False, "Generic Error")
-
-    def verify_job_settings(self, build_id: str, new_settings: dict[str, str]) -> tuple[bool, str]:
-        PLUGIN_MAP = {
-            "scylla-cluster-tests": self._verify_sct_settings,
-            # for now they match
-            "sirenada": self._verify_sct_settings,
-            "driver-matrix-tests": self._verify_sct_settings,
-        }
-        test: ArgusTest = ArgusTest.get(build_system_id=build_id)
-        plugin_name = test.plugin_name
-
-        validated, message = PLUGIN_MAP.get(plugin_name, lambda _: (True, ""))(new_settings)
-
-        return {
-            "validated": validated,
-            "message": message,
-        }
-
-    def get_advanced_settings(self, build_id: str):
-        test: ArgusTest = ArgusTest.get(build_system_id=build_id)
-        plugin_name = test.plugin_name
-
-        if not (plugin_settings := self.SETTINGS_CONFIG_MAP.get(plugin_name)):
-            return {}
-
-        settings = {}
-        raw_config = self._jenkins.get_job_config(name=build_id)
-        config = ET.fromstring(raw_config)
-
-        for setting, xpath in plugin_settings.items():
-            value = config.find(xpath)
-            settings[setting] = value.text
-
-        return settings
-
-    def adjust_job_settings(self, build_id: str, plugin_name: str, settings: dict[str, str]):
-        xpath_map = self.SETTINGS_CONFIG_MAP.get(plugin_name)
-        if not xpath_map:
-            return
-        
-        config = self._jenkins.get_job_config(name=build_id)
-        xml = ET.fromstring(config)
-        for setting, value in settings.items():
-            element = xml.find(xpath_map[setting])
-            element.text = value
-
-        adjusted_config = ET.tostring(xml, encoding="unicode")
-        self._jenkins.reconfig_job(name=build_id, config_xml=adjusted_config)
-
-    def clone_job(self, current_test_id: str, new_name: str, target: str, group: str, advanced_settings: bool | dict[str, str]):
-        cloned_test: ArgusTest = ArgusTest.get(id=current_test_id)
-        target_release: ArgusRelease = ArgusRelease.get(id=target)
-        target_group: ArgusGroup = ArgusGroup.get(id=group)
-
-        if target_group.id == cloned_test.id and new_name == cloned_test.name:
-            raise JenkinsServiceError("Unable to clone: source and destination are the same")
-
-        if not target_group.build_system_id:
-            raise JenkinsServiceError("Unable to clone: target group is missing jenkins folder path")
-
-        jenkins_new_build_id = f"{target_group.build_system_id}/{new_name}"
-
-        new_test = ArgusTest()
-        new_test.name = new_name
-        new_test.build_system_id = jenkins_new_build_id
-        new_test.group_id = target_group.id
-        new_test.release_id = target_release.id
-        new_test.plugin_name = cloned_test.plugin_name
-
-        old_config = self._jenkins.get_job_config(name=cloned_test.build_system_id)
-        LOGGER.info(old_config)
-        xml = ET.fromstring(old_config)
-        display_name = xml.find("displayName")
-        if display_name:
-            display_name.text = new_name
-        new_config = ET.tostring(xml, encoding="unicode")
-        self._jenkins.create_job(name=jenkins_new_build_id, config_xml=new_config)
-        new_job_info = self._jenkins.get_job_info(name=jenkins_new_build_id)
-        new_test.build_system_url = new_job_info["url"]
-        new_test.save()
-
-        if advanced_settings:
-            self.adjust_job_settings(build_id=jenkins_new_build_id, plugin_name=new_test.plugin_name, settings=advanced_settings)
-
-        return {
-            "new_job": new_job_info,
-            "new_entity": new_test,
-        }
-
-    def clone_build_job(self, build_id: str, params: dict[str, str]):
-        queue_item = self.build_job(build_id=build_id, params=params)
-        return {
-            "queueItem": queue_item,
-        }
-
-    def build_job(self, build_id: str, params: dict, user_override: str = None):
-        queue_number = self._jenkins.build_job(build_id, {
-            **params,
-            # use the user's email as the default value for the requested by user parameter,
-            # so it would align with how SCT default works, on runs not trigger by argus
-            self.RESERVED_PARAMETER_NAME: g.user.email.split('@')[0] if not user_override else user_override
-        })
-        return queue_number
-
-    def get_queue_info(self, queue_item: int):
-        build_info = self._jenkins.get_queue_item(queue_item)
-        LOGGER.info("%s", build_info)
-        executable = build_info.get("executable")
-        if executable:
-            return executable
-        else:
-            return {
-                "why": build_info["why"],
-                "inQueueSince": build_info["inQueueSince"],
-                "taskUrl": build_info["task"]["url"],
-            }