argus-alm 0.14.2__py3-none-any.whl → 0.15.2__py3-none-any.whl

argus/backend/service/build_system_monitor.py

@@ -0,0 +1,185 @@
+import logging
+from abc import ABC, abstractmethod
+import jenkins
+import click
+import re
+from flask import current_app
+from flask.cli import with_appcontext
+
+from argus.backend.db import ScyllaCluster
+from argus.backend.models.web import ArgusRelease, ArgusGroup, ArgusTest, ArgusTestException
+from argus.backend.service.release_manager import ReleaseManagerService
+
+LOGGER = logging.getLogger(__name__)
+
+
+class ArgusTestsMonitor(ABC):
+    BUILD_SYSTEM_FILTERED_PREFIXES = [
+
+    ]
+
+    def __init__(self) -> None:
+        self._cluster = ScyllaCluster.get()
+        self._existing_releases = list(ArgusRelease.objects().limit(None))
+        self._existing_groups = list(ArgusGroup.objects().limit(None))
+        self._existing_tests = list(ArgusTest.objects().limit(None))
+        self._filtered_groups: list[str] = self.BUILD_SYSTEM_FILTERED_PREFIXES
+
+    def create_release(self, release_name):
+        release = ArgusRelease()
+        release.name = release_name
+        release.save()
+
+        return release
+
+    def create_group(self, release: ArgusRelease, group_name: str, build_id: str, group_pretty_name: str | None = None):
+        group = ArgusGroup()
+        group.release_id = release.id
+        group.name = group_name
+        group.build_system_id = build_id
+        if group_pretty_name:
+            group.pretty_name = group_pretty_name
+        group.save()
+
+        return group
+
+    def create_test(self, release: ArgusRelease, group: ArgusGroup,
+                    test_name: str, build_id: str, build_url: str) -> ArgusTest:
+        test = ArgusTest()
+        test.name = test_name
+        test.group_id = group.id
+        test.release_id = release.id
+        test.build_system_id = build_id
+        test.build_system_url = build_url
+        test.validate_build_system_id()
+        test.save()
+        ReleaseManagerService().move_test_runs(test)
+
+        return test
+
+    @abstractmethod
+    def collect(self):
+        raise NotImplementedError()
+
+    def check_filter(self, group_name: str) -> bool:
+        for prefix in self._filtered_groups:
+            if prefix.lower() in group_name.lower():
+                return False
+
+        return True
+
+
+class JenkinsMonitor(ArgusTestsMonitor):
+
+    BUILD_SYSTEM_FILTERED_PREFIXES = [
+        "releng",
+    ]
+
+    JENKINS_MONITORED_RELEASES = [
+        r"^scylla-master$",
+        r"^scylla-staging$",
+        r"^scylla-\d+\.\d+$",
+        r"^manager-3.\d+$",
+        r"^scylla-operator/operator-master$",
+        r"^scylla-operator/operator-\d+.\d+$",
+        r"^scylla-enterprise$",
+        r"^enterprise-20\d{2}\.\d+$",
+        r"^siren-tests$",
+    ]
+
+    def __init__(self) -> None:
+        super().__init__()
+        self._jenkins = jenkins.Jenkins(url=current_app.config["JENKINS_URL"],
+                                        username=current_app.config["JENKINS_USER"],
+                                        password=current_app.config["JENKINS_API_TOKEN"])
+        self._monitored_releases = self.JENKINS_MONITORED_RELEASES
+
+    def _check_release_name(self, release_name: str):
+        return any(re.match(pattern, release_name, re.IGNORECASE) for pattern in self._monitored_releases)
+
+    def collect(self):
+        click.echo("Collecting new tests from jenkins")
+        all_jobs = self._jenkins.get_all_jobs()
+        all_monitored_folders = [job for job in all_jobs if self._check_release_name(job["fullname"])]
+        LOGGER.info("Will collect %s", [f["fullname"] for f in all_monitored_folders])
+
+        for release in all_monitored_folders:
+            LOGGER.info("Processing release %s", release["name"])
+            try:
+                saved_release = ArgusRelease.get(name=release["name"])
+                LOGGER.info("Release %s exists", release["name"])
+            except ArgusRelease.DoesNotExist:
+                LOGGER.warning("Release %s does not exist, creating...", release["name"])
+                saved_release = self.create_release(release["name"])
+                self._existing_releases.append(saved_release)
+
+            try:
+                groups = self.collect_groups_for_release(release["jobs"])
+            except KeyError:
+                LOGGER.error("Empty release!\n %s", release)
+                continue
+            folder_stack = [dict(parent_name="", parent_display_name="", group=g) for g in reversed(groups)]
+            root_folder = {
+                "parent_name": "",
+                "parent_display_name": "",
+                "group":  {
+                    "name": f"{release['fullname']}-root",
+                    "displayName": "-- root directory --",
+                    "fullname": release["fullname"],
+                    "jobs": self.collect_root_folder_jobs(release["jobs"]),
+                }
+            }
+            folder_stack.append(root_folder)
+            while len(folder_stack) != 0:
+                group_dict = folder_stack.pop()
+                group = group_dict["group"]
+                LOGGER.info("Processing group %s for release %s", group["name"], saved_release.name)
+                try:
+                    group_name = group["name"] if not group_dict["parent_name"] else f"{group_dict['parent_name']}-{group['name']}"
+                    saved_group = filter(lambda g: g.build_system_id == group["fullname"], self._existing_groups)
+                    saved_group = next(saved_group)
+                    LOGGER.info("Group %s already exists. (id: %s)", saved_group.build_system_id, saved_group.id)
+                except StopIteration:
+                    LOGGER.warning(
+                        "Group %s for release %s doesn't exist, creating...", group_name, saved_release.name)
+                    try:
+                        display_name = group.get("displayName", self._jenkins.get_job_info(
+                            name=group["fullname"])["displayName"])
+                        display_name = display_name if not group_dict[
+                            "parent_display_name"] else f"{group_dict['parent_display_name']} - {display_name}"
+                    except Exception:
+                        display_name = None
+
+                    saved_group = self.create_group(saved_release, group_name, group["fullname"], display_name)
+                    self._existing_groups.append(saved_group)
+
+                for job in group["jobs"]:
+                    LOGGER.info("Processing job %s for release %s and group %s",
+                                job["fullname"], saved_group.name, saved_release.name)
+                    saved_test = None
+                    if "Folder" in job["_class"]:
+                        folder_stack.append(dict(parent_name=saved_group.name,
+                                            parent_display_name=saved_group.pretty_name, group=job))
+                    if "WorkflowJob" in job["_class"]:
+                        try:
+                            saved_test = filter(lambda t: t.build_system_id == job["fullname"], self._existing_tests)
+                            saved_test = next(saved_test)
+                            LOGGER.info("Test %s already exists. (id: %s)", saved_test.build_system_id, saved_test.id)
+                        except StopIteration:
+                            LOGGER.warning("Test %s for release %s (group %s) doesn't exist, creating...",
+                                           job["name"], saved_release.name, saved_group.name)
+                            try:
+                                saved_test = self.create_test(
+                                    saved_release, saved_group, job["name"], job["fullname"], job["url"])
+                                self._existing_tests.append(saved_test)
+                            except ArgusTestException:
+                                LOGGER.error("Unable to create test for build_id %s", job["fullname"], exc_info=True)
+
+    def collect_groups_for_release(self, jobs):
+        groups = [folder for folder in jobs if "Folder" in folder["_class"]]
+        groups = [group for group in groups if self.check_filter(group["name"])]
+
+        return groups
+
+    def collect_root_folder_jobs(self, jobs):
+        return [job for job in jobs if "WorkflowJob" in job["_class"]]

argus/backend/service/client_service.py

@@ -0,0 +1,127 @@
+from dataclasses import asdict
+from datetime import datetime
+from uuid import UUID
+
+from argus.backend.db import ScyllaCluster
+from argus.backend.error_handlers import DataValidationError
+from argus.backend.models.result import ArgusGenericResultMetadata, ArgusGenericResultData
+from argus.backend.plugins.core import PluginModelBase
+from argus.backend.plugins.loader import AVAILABLE_PLUGINS
+from argus.backend.service.results_service import ResultsService, Cell
+from argus.common.enums import TestStatus
+
+
+class ClientException(Exception):
+    pass
+
+
+class ClientService:
+    PLUGINS = {name: plugin.model for name, plugin in AVAILABLE_PLUGINS.items()}
+
+    def __init__(self) -> None:
+        self.cluster = ScyllaCluster.get()
+
+    def get_model(self, run_type: str) -> PluginModelBase:
+        cls = self.PLUGINS.get(run_type)
+        if not cls:
+            raise ClientException(f"Unsupported run type: {run_type}", run_type)
+        return cls
+
+    def submit_run(self, run_type: str, request_data: dict) -> str:
+        model = self.get_model(run_type)
+        model.submit_run(request_data=request_data)
+
+        return "Created"
+
+    def get_run(self, run_type: str, run_id: str):
+        model = self.get_model(run_type)
+        try:
+            run = model.get(id=run_id)
+        except model.DoesNotExist:
+            return None
+        return run
+
+    def heartbeat(self, run_type: str, run_id: str) -> int:
+        model = self.get_model(run_type)
+        run = model.load_test_run(UUID(run_id))
+        run.update_heartbeat()
+        run.save()
+        return run.heartbeat
+
+    def get_run_status(self, run_type: str, run_id: str) -> str:
+        model = self.get_model(run_type)
+        run = model.load_test_run(UUID(run_id))
+        return run.status
+
+    def update_run_status(self, run_type: str, run_id: str, new_status: str) -> str:
+        model = self.get_model(run_type)
+        run = model.load_test_run(UUID(run_id))
+        run.change_status(new_status=TestStatus(new_status))
+        run.save()
+
+        return run.status
+
+    def submit_product_version(self, run_type: str, run_id: str, version: str) -> str:
+        model = self.get_model(run_type)
+        run = model.load_test_run(UUID(run_id))
+        run.submit_product_version(version)
+        run.save()
+
+        return "Submitted"
+
+    def submit_logs(self, run_type: str, run_id: str, logs: list[dict]) -> str:
+        model = self.get_model(run_type)
+        run = model.load_test_run(UUID(run_id))
+        run.submit_logs(logs)
+        run.save()
+
+        return "Submitted"
+
+    def finish_run(self, run_type: str, run_id: str, payload: dict | None = None) -> str:
+        model = self.get_model(run_type)
+        run = model.load_test_run(UUID(run_id))
+        run.finish_run(payload)
+        run.save()
+
+        return "Finalized"
+
+    def submit_results(self, run_type: str, run_id: str, results: dict) -> dict[str, str]:
+        model = self.get_model(run_type)
+        try:
+            run = model.load_test_run(UUID(run_id))
+        except model.DoesNotExist:
+            return {"status": "error", "response": {
+                "exception": "DoesNotExist",
+                "arguments": [run_id]
+            }}
+        table_name = results["meta"]["name"]
+        results_service = ResultsService()
+        cells = [Cell(**cell) for cell in results["results"]]
+        table_metadata = results_service.get_table_metadata(test_id=run.test_id, table_name=table_name)
+        if table_metadata:
+            table_metadata = table_metadata.update_if_changed(results["meta"])
+        else:
+            table_metadata = ArgusGenericResultMetadata(test_id=run.test_id, **results["meta"])
+            table_metadata.save()
+        if results.get("sut_timestamp", 0) == 0:
+            results["sut_timestamp"] = run.sut_timestamp(results.get(
+                'sut_package_name', 'scylla-server'))  # automatic sut_timestamp
+        results["sut_timestamp"] = datetime.fromtimestamp(results["sut_timestamp"])
+        best_results = results_service.update_best_results(test_id=run.test_id, table_name=table_name, table_metadata=table_metadata,
+                                                           cells=cells, run_id=run_id)
+        table_name = results["meta"]["name"]
+        sut_timestamp = results["sut_timestamp"]
+        result_failed = False
+        for cell in cells:
+            cell.update_cell_status_based_on_rules(table_metadata, best_results)
+            if cell.status == "ERROR":
+                result_failed = True
+            ArgusGenericResultData(test_id=run.test_id,
+                                   run_id=run.id,
+                                   name=table_name,
+                                   sut_timestamp=sut_timestamp,
+                                   **asdict(cell)
+                                   ).save()
+        if result_failed:
+            raise DataValidationError()
+        return {"status": "ok", "message": "Results submitted"}

argus/backend/service/event_service.py

@@ -0,0 +1,18 @@
+from datetime import datetime
+import json
+from argus.backend.models.web import ArgusEvent, ArgusEventTypes
+
+
+class EventService:
+    @staticmethod
+    def create_run_event(kind: ArgusEventTypes, body: dict, user_id=None, run_id=None, release_id=None, group_id=None, test_id=None):
+        event = ArgusEvent()
+        event.release_id = release_id
+        event.group_id = group_id
+        event.test_id = test_id
+        event.user_id = user_id
+        event.run_id = run_id
+        event.body = json.dumps(body, ensure_ascii=True, separators=(',', ':'))
+        event.kind = kind.value
+        event.created_at = datetime.utcnow()
+        event.save()

argus/backend/service/github_service.py

@@ -0,0 +1,233 @@
+import re
+import logging
+from collections import defaultdict
+from datetime import UTC, datetime
+from functools import reduce
+from uuid import UUID
+from flask import current_app, g
+from github import Github, Auth
+
+from argus.backend.models.runtime_store import RuntimeStore
+from argus.backend.models.web import ArgusEventTypes, ArgusTest, ArgusUserView
+from argus.backend.models.github_issue import GithubIssue, IssueAssignee, IssueLink, IssueLabel
+from argus.backend.plugins.core import PluginInfoBase
+from argus.backend.plugins.loader import AVAILABLE_PLUGINS
+from argus.backend.service.event_service import EventService
+from argus.backend.util.common import chunk
+
+LOGGER = logging.getLogger(__name__)
+
+
+class GithubService:
+    LAST_RAN_KEY = "github_service_last_issue_refresh"
+
+    plugins = AVAILABLE_PLUGINS
+
+    def __init__(self):
+        auth = Auth.Token(token=self.get_installation_token())
+        self.gh = Github(auth=auth, per_page=1000)
+
+    def get_plugin(self, plugin_name: str) -> PluginInfoBase | None:
+        return self.plugins.get(plugin_name)
+
+    def get_installation_token(self):
+        self._refresh_installation_token()
+        return current_app.config.get("GITHUB_ACCESS_TOKEN")
+
+    def _refresh_installation_token(self):
+        # TODO: To be replaced by JWT refreshing logic once we have Github App in place
+        pass
+
+    def refresh_stale_issues(self):
+        try:
+            last_ran = RuntimeStore.get(key=self.LAST_RAN_KEY)
+        except RuntimeStore.DoesNotExist:
+            last_ran = RuntimeStore()
+            last_ran.key = self.LAST_RAN_KEY
+            last_ran.value = datetime(year=2020, month=1, day=1, hour=0, minute=0, tzinfo=UTC)
+            last_ran.save()
+
+        LOGGER.info("Starting Github Issue sync...")
+        check_time = datetime.now(tz=UTC)
+
+        all_issues: list[GithubIssue] = list(GithubIssue.all())
+        issues_by_identifier = {
+            f"{issue.owner.lower()}/{issue.repo.lower()}#{issue.number}": issue for issue in all_issues}
+        touch_count = 0
+
+        unique_repos = {f"{issue.owner}/{issue.repo}" for issue in all_issues}
+        for idx, repo in enumerate(unique_repos):
+            LOGGER.info("[%s/%s] Fetching %s...", idx + 1, len(unique_repos), repo)
+            repo = self.gh.get_repo(repo)
+            issues = repo.get_issues(since=last_ran.value, state="all", direction="desc", sort="created")
+            for issue_idx, issue in enumerate(issues):
+                match = re.match(
+                    r"http(s)?://(www\.)?github\.com/(?P<owner>[\w\d]+)/"
+                    r"(?P<repo>[\w\d\-_]+)/(?P<type>issues|pull)/(?P<issue_number>\d+)(/)?",
+                    issue.html_url,
+                )
+
+                identifier = f"{match.group('owner').lower()}/{match.group('repo').lower()}#{match.group('issue_number')}"
+                issue_to_update = issues_by_identifier.get(identifier)
+                if not issue_to_update:
+                    LOGGER.debug("[%s/%s] No issue found for %s...", issue_idx + 1, "?", identifier)
+                    continue
+                LOGGER.debug("[%s/%s] Refreshing %s...", issue_idx + 1, "?", identifier)
+                issue_to_update.title = issue.title
+                issue_to_update.state = issue.state
+                issue_to_update.labels = [IssueLabel(
+                    id=label.id, name=label.name, color=label.color, description=label.description) for label in issue.labels]
+                issue_to_update.assignees = [IssueAssignee(
+                    login=assignee.login, html_url=assignee.html_url) for assignee in issue.assignees]
+                issue_to_update.save()
+                touch_count += 1
+
+        LOGGER.info("Finished. Found %s out of %s issues", touch_count, len(all_issues))
+        last_ran.value = check_time
+        last_ran.save()
+
+    def submit_github_issue(self, issue_url: str, test_id: UUID, run_id: UUID):
+        match = re.match(
+            r"http(s)?://(www\.)?github\.com/(?P<owner>[\w\d]+)/"
+            r"(?P<repo>[\w\d\-_]+)/(?P<type>issues|pull)/(?P<issue_number>\d+)(/)?",
+            issue_url,
+        )
+        if not match:
+            raise Exception("URL doesn't match Github schema")
+
+        test: ArgusTest = ArgusTest.get(id=test_id)
+        plugin = self.get_plugin(plugin_name=test.plugin_name)
+        run = plugin.model.get(id=run_id)
+
+        existing = True
+        try:
+            issue = GithubIssue.get(url=issue_url)
+        except:
+            issue = None
+            existing = False
+        if not issue:
+            repo_id = f"{match.group('owner')}/{match.group('repo')}"
+            remote_repo = self.gh.get_repo(repo_id)
+            remote_issue = remote_repo.get_issue(int(match.group("issue_number")))
+
+            issue = GithubIssue()
+            issue.user_id = g.user.id
+            issue.type = match.group("type")
+            issue.owner = remote_issue.repository.owner.name
+            issue.repo = remote_issue.repository.name
+            issue.number = remote_issue.number
+            issue.state = remote_issue.state
+            issue.title = remote_issue.title
+            issue.url = issue_url
+            issue.repo_identifier = repo_id
+            for label in remote_issue.labels:
+                l = IssueLabel()
+                l.id = label.id
+                l.name = label.name
+                l.description = label.description
+                l.color = label.color
+                issue.labels.append(l)
+
+            for assignee in remote_issue.assignees:
+                a = IssueAssignee()
+                a.login = assignee.login
+                a.html_url = assignee.html_url
+                issue.assignees.append(a)
+
+            issue.save()
+
+        link = IssueLink()
+        link.run_id = run.id
+        link.issue_id = issue.id
+        link.release_id = test.release_id
+        link.test_id = test.id
+        link.group_id = test.group_id
+
+        link.save()
+
+        EventService.create_run_event(
+            kind=ArgusEventTypes.TestRunIssueAdded,
+            body={
+                "message": f"An issue titled \"{{title}}\" was {'attached' if existing else 'added'} by {{username}}",
+                "username": g.user.username,
+                "url": issue_url,
+                "title": issue.title,
+                "state": issue.state,
+            },
+            user_id=g.user.id,
+            run_id=link.run_id,
+            release_id=link.release_id,
+            group_id=link.group_id,
+            test_id=link.test_id
+        )
+
+        response = {
+            **dict(list(issue.items())),
+            "title": issue.title,
+            "state": issue.state,
+        }
+
+        return response
+
+    def _get_github_issues_for_view(self, view_id: UUID | str) -> list[IssueLink]:
+        view: ArgusUserView = ArgusUserView.get(id=view_id)
+        links = []
+        for batch in chunk(view.tests):
+            links.extend(IssueLink.filter(test_id__in=batch).allow_filtering().all())
+
+        return links
+
+    def get_github_issues(self, filter_key: str, filter_id: UUID, aggregate_by_issue: bool = False) -> dict:
+        if filter_key not in ["release_id", "group_id", "test_id", "run_id", "user_id", "view_id"]:
+            raise Exception(
+                "filter_key can only be one of: \"release_id\", \"group_id\", \"test_id\", \"run_id\", \"user_id\", \"view_id\""
+            )
+        if filter_key == "view_id":
+            links = list(self._get_github_issues_for_view(filter_id))
+        else:
+            links = list(IssueLink.filter(**{filter_key: filter_id}).allow_filtering().all())
+        issues = reduce(lambda acc, link: acc[link.issue_id].append(link) or acc, links, defaultdict(list))
+        resolved_issues = []
+        for batch in chunk(issues.keys()):
+            resolved_issues.extend(GithubIssue.filter(id__in=batch).all())
+        if aggregate_by_issue:
+            response = []
+            for issue in resolved_issues:
+                issue_dict = dict(issue.items())
+                issue_dict["links"] = issues[issue.id]
+                response.append(issue_dict)
+
+        else:
+            response = [dict(issue.items()) for issue in resolved_issues]
+        return response
+
+    def delete_github_issue(self, issue_id: UUID, run_id: UUID) -> dict:
+        issue: GithubIssue = GithubIssue.get(id=issue_id)
+        links = list(IssueLink.filter(issue_id=issue_id).allow_filtering().all())
+        link: IssueLink = IssueLink.get(run_id=run_id, issue_id=issue_id)
+        remaining_links = len(list(filter(lambda l: l.run_id != link.run_id and link.issue_id != issue_id, links)))
+
+        EventService.create_run_event(
+            kind=ArgusEventTypes.TestRunIssueRemoved,
+            body={
+                "message": "An issue titled \"{title}\" was removed by {username} from \"{run_id}\"",
+                "username": g.user.username,
+                "url": issue.url,
+                "title": issue.title,
+                "state": issue.state,
+                "run_id": run_id,
+            },
+            user_id=g.user.id,
+            run_id=link.run_id,
+            release_id=link.release_id,
+            group_id=link.group_id,
+            test_id=link.test_id
+        )
+
+        link.delete()
+        if remaining_links == 0:
+            issue.delete()
+
+        return {
+            "deleted": issue_id if remaining_links == 0 else (link.run_id, link.issue_id)
+        }