argus-alm 0.14.2__py3-none-any.whl → 0.15.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- argus/_version.py +21 -0
- argus/backend/.gitkeep +0 -0
- argus/backend/__init__.py +0 -0
- argus/backend/cli.py +57 -0
- argus/backend/controller/__init__.py +0 -0
- argus/backend/controller/admin.py +20 -0
- argus/backend/controller/admin_api.py +355 -0
- argus/backend/controller/api.py +589 -0
- argus/backend/controller/auth.py +67 -0
- argus/backend/controller/client_api.py +109 -0
- argus/backend/controller/main.py +316 -0
- argus/backend/controller/notification_api.py +72 -0
- argus/backend/controller/notifications.py +13 -0
- argus/backend/controller/planner_api.py +194 -0
- argus/backend/controller/team.py +129 -0
- argus/backend/controller/team_ui.py +19 -0
- argus/backend/controller/testrun_api.py +513 -0
- argus/backend/controller/view_api.py +188 -0
- argus/backend/controller/views_widgets/__init__.py +0 -0
- argus/backend/controller/views_widgets/graphed_stats.py +54 -0
- argus/backend/controller/views_widgets/graphs.py +68 -0
- argus/backend/controller/views_widgets/highlights.py +135 -0
- argus/backend/controller/views_widgets/nemesis_stats.py +26 -0
- argus/backend/controller/views_widgets/summary.py +43 -0
- argus/backend/db.py +98 -0
- argus/backend/error_handlers.py +41 -0
- argus/backend/events/event_processors.py +34 -0
- argus/backend/models/__init__.py +0 -0
- argus/backend/models/argus_ai.py +24 -0
- argus/backend/models/github_issue.py +60 -0
- argus/backend/models/plan.py +24 -0
- argus/backend/models/result.py +187 -0
- argus/backend/models/runtime_store.py +58 -0
- argus/backend/models/view_widgets.py +25 -0
- argus/backend/models/web.py +403 -0
- argus/backend/plugins/__init__.py +0 -0
- argus/backend/plugins/core.py +248 -0
- argus/backend/plugins/driver_matrix_tests/controller.py +66 -0
- argus/backend/plugins/driver_matrix_tests/model.py +429 -0
- argus/backend/plugins/driver_matrix_tests/plugin.py +21 -0
- argus/backend/plugins/driver_matrix_tests/raw_types.py +62 -0
- argus/backend/plugins/driver_matrix_tests/service.py +61 -0
- argus/backend/plugins/driver_matrix_tests/udt.py +42 -0
- argus/backend/plugins/generic/model.py +86 -0
- argus/backend/plugins/generic/plugin.py +15 -0
- argus/backend/plugins/generic/types.py +14 -0
- argus/backend/plugins/loader.py +39 -0
- argus/backend/plugins/sct/controller.py +224 -0
- argus/backend/plugins/sct/plugin.py +37 -0
- argus/backend/plugins/sct/resource_setup.py +177 -0
- argus/backend/plugins/sct/service.py +682 -0
- argus/backend/plugins/sct/testrun.py +288 -0
- argus/backend/plugins/sct/udt.py +100 -0
- argus/backend/plugins/sirenada/model.py +118 -0
- argus/backend/plugins/sirenada/plugin.py +16 -0
- argus/backend/service/admin.py +26 -0
- argus/backend/service/argus_service.py +696 -0
- argus/backend/service/build_system_monitor.py +185 -0
- argus/backend/service/client_service.py +127 -0
- argus/backend/service/event_service.py +18 -0
- argus/backend/service/github_service.py +233 -0
- argus/backend/service/jenkins_service.py +269 -0
- argus/backend/service/notification_manager.py +159 -0
- argus/backend/service/planner_service.py +608 -0
- argus/backend/service/release_manager.py +229 -0
- argus/backend/service/results_service.py +690 -0
- argus/backend/service/stats.py +610 -0
- argus/backend/service/team_manager_service.py +82 -0
- argus/backend/service/test_lookup.py +172 -0
- argus/backend/service/testrun.py +489 -0
- argus/backend/service/user.py +308 -0
- argus/backend/service/views.py +219 -0
- argus/backend/service/views_widgets/__init__.py +0 -0
- argus/backend/service/views_widgets/graphed_stats.py +180 -0
- argus/backend/service/views_widgets/highlights.py +374 -0
- argus/backend/service/views_widgets/nemesis_stats.py +34 -0
- argus/backend/template_filters.py +27 -0
- argus/backend/tests/__init__.py +0 -0
- argus/backend/tests/client_service/__init__.py +0 -0
- argus/backend/tests/client_service/test_submit_results.py +79 -0
- argus/backend/tests/conftest.py +180 -0
- argus/backend/tests/results_service/__init__.py +0 -0
- argus/backend/tests/results_service/test_best_results.py +178 -0
- argus/backend/tests/results_service/test_cell.py +65 -0
- argus/backend/tests/results_service/test_chartjs_additional_functions.py +259 -0
- argus/backend/tests/results_service/test_create_chartjs.py +220 -0
- argus/backend/tests/results_service/test_result_metadata.py +100 -0
- argus/backend/tests/results_service/test_results_service.py +203 -0
- argus/backend/tests/results_service/test_validation_rules.py +213 -0
- argus/backend/tests/view_widgets/__init__.py +0 -0
- argus/backend/tests/view_widgets/test_highlights_api.py +532 -0
- argus/backend/util/common.py +65 -0
- argus/backend/util/config.py +38 -0
- argus/backend/util/encoders.py +56 -0
- argus/backend/util/logsetup.py +80 -0
- argus/backend/util/module_loaders.py +30 -0
- argus/backend/util/send_email.py +91 -0
- argus/client/base.py +1 -3
- argus/client/driver_matrix_tests/cli.py +17 -8
- argus/client/generic/cli.py +4 -2
- argus/client/generic/client.py +1 -0
- argus/client/generic_result.py +48 -9
- argus/client/sct/client.py +1 -3
- argus/client/sirenada/client.py +4 -1
- argus/client/tests/__init__.py +0 -0
- argus/client/tests/conftest.py +19 -0
- argus/client/tests/test_package.py +45 -0
- argus/client/tests/test_results.py +224 -0
- argus/common/sct_types.py +3 -0
- argus/common/sirenada_types.py +1 -1
- {argus_alm-0.14.2.dist-info → argus_alm-0.15.2.dist-info}/METADATA +43 -19
- argus_alm-0.15.2.dist-info/RECORD +122 -0
- {argus_alm-0.14.2.dist-info → argus_alm-0.15.2.dist-info}/WHEEL +2 -1
- argus_alm-0.15.2.dist-info/entry_points.txt +3 -0
- argus_alm-0.15.2.dist-info/top_level.txt +1 -0
- argus_alm-0.14.2.dist-info/RECORD +0 -20
- argus_alm-0.14.2.dist-info/entry_points.txt +0 -4
- {argus_alm-0.14.2.dist-info → argus_alm-0.15.2.dist-info/licenses}/LICENSE +0 -0
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
from functools import partial
|
|
4
|
+
import re
|
|
5
|
+
from urllib.parse import unquote
|
|
6
|
+
from typing import Any, Callable
|
|
7
|
+
from uuid import UUID
|
|
8
|
+
|
|
9
|
+
from cassandra.cqlengine.models import Model
|
|
10
|
+
from argus.backend.models.web import ArgusGroup, ArgusRelease, ArgusTest
|
|
11
|
+
from argus.backend.plugins.core import PluginModelBase
|
|
12
|
+
from argus.backend.plugins.loader import all_plugin_models
|
|
13
|
+
from argus.backend.util.common import get_build_number
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class TestLookup:
|
|
17
|
+
ADD_ALL_ID = UUID("db6f33b2-660b-4639-ba7f-79725ef96616")
|
|
18
|
+
|
|
19
|
+
@classmethod
|
|
20
|
+
def index_mapper(cls, item: Model, type="test"):
|
|
21
|
+
mapped = dict(item)
|
|
22
|
+
mapped["type"] = type
|
|
23
|
+
return mapped
|
|
24
|
+
|
|
25
|
+
@classmethod
|
|
26
|
+
def explode_group(cls, group_id: UUID | str):
|
|
27
|
+
group = ArgusGroup.get(id=group_id)
|
|
28
|
+
release = ArgusRelease.get(id=group.release_id)
|
|
29
|
+
tests = ArgusTest.filter(group_id=group.id).all()
|
|
30
|
+
|
|
31
|
+
exploded = []
|
|
32
|
+
for test in tests:
|
|
33
|
+
test = cls.index_mapper(test)
|
|
34
|
+
test["group"] = group.pretty_name or group.name
|
|
35
|
+
test["release"] = release.name
|
|
36
|
+
test["name"] = test["pretty_name"] or test["name"]
|
|
37
|
+
exploded.append(test)
|
|
38
|
+
|
|
39
|
+
return exploded
|
|
40
|
+
|
|
41
|
+
@classmethod
|
|
42
|
+
def find_run(self, run_id: UUID) -> PluginModelBase | None:
|
|
43
|
+
for model in all_plugin_models():
|
|
44
|
+
try:
|
|
45
|
+
return model.get(id=run_id)
|
|
46
|
+
except model.DoesNotExist:
|
|
47
|
+
pass
|
|
48
|
+
return None
|
|
49
|
+
|
|
50
|
+
@classmethod
|
|
51
|
+
def query_to_uuid(cls, query: str) -> UUID | None:
|
|
52
|
+
try:
|
|
53
|
+
uuid = UUID(query.strip())
|
|
54
|
+
return uuid
|
|
55
|
+
except ValueError:
|
|
56
|
+
return None
|
|
57
|
+
|
|
58
|
+
@classmethod
|
|
59
|
+
def resolve_run_test(cls, test_id: UUID) -> ArgusTest:
|
|
60
|
+
try:
|
|
61
|
+
test = ArgusTest.get(id=test_id)
|
|
62
|
+
return test
|
|
63
|
+
except ArgusTest.DoesNotExist:
|
|
64
|
+
return None
|
|
65
|
+
|
|
66
|
+
@classmethod
|
|
67
|
+
def resolve_run_group(cls, group_id: UUID) -> ArgusGroup:
|
|
68
|
+
try:
|
|
69
|
+
group = ArgusGroup.get(id=group_id)
|
|
70
|
+
return group
|
|
71
|
+
except ArgusGroup.DoesNotExist:
|
|
72
|
+
return None
|
|
73
|
+
|
|
74
|
+
@classmethod
|
|
75
|
+
def resolve_run_release(cls, run_test_id: UUID) -> ArgusRelease:
|
|
76
|
+
try:
|
|
77
|
+
release = ArgusRelease.get(id=run_test_id)
|
|
78
|
+
return release
|
|
79
|
+
except ArgusRelease.DoesNotExist:
|
|
80
|
+
return None
|
|
81
|
+
|
|
82
|
+
@classmethod
|
|
83
|
+
def make_single_run_response(cls, run_id: UUID) -> list[dict[str, Any]]:
|
|
84
|
+
run = cls.find_run(run_id)
|
|
85
|
+
if run:
|
|
86
|
+
run = dict(run.items())
|
|
87
|
+
run["type"] = "run"
|
|
88
|
+
run["test"] = dict(cls.resolve_run_test(run["test_id"]).items()) if run["test_id"] else None
|
|
89
|
+
if run["test"]:
|
|
90
|
+
name = run["test"]["name"]
|
|
91
|
+
run["group"] = dict(cls.resolve_run_group(run["group_id"]).items())if run["group_id"] else None
|
|
92
|
+
run["release"] = dict(cls.resolve_run_release(run["release_id"]).items()) if run["release_id"] else None
|
|
93
|
+
run["build_number"] = get_build_number(run["build_job_url"])
|
|
94
|
+
run["name"] = f"{name}#{run['build_number']}"
|
|
95
|
+
|
|
96
|
+
return [run]
|
|
97
|
+
|
|
98
|
+
return []
|
|
99
|
+
|
|
100
|
+
@classmethod
|
|
101
|
+
def test_lookup(cls, query: str, release_id: UUID | str = None):
|
|
102
|
+
if uuid := cls.query_to_uuid(query):
|
|
103
|
+
return cls.make_single_run_response(uuid)
|
|
104
|
+
|
|
105
|
+
def check_visibility(entity: dict):
|
|
106
|
+
if entity["type"] == "release" and release_id:
|
|
107
|
+
return False
|
|
108
|
+
if not entity["enabled"]:
|
|
109
|
+
return False
|
|
110
|
+
if entity.get("group") and not entity["group"]["enabled"]:
|
|
111
|
+
return False
|
|
112
|
+
if entity.get("release") and not entity["release"]["enabled"]:
|
|
113
|
+
return False
|
|
114
|
+
return True
|
|
115
|
+
|
|
116
|
+
def facet_extraction(query: str) -> str:
|
|
117
|
+
extractor = re.compile(r"(?:(?P<name>(?:release|group|type)):(?P<value>\"?[\w\d\.\-]*\"?))")
|
|
118
|
+
facets = re.findall(extractor, query)
|
|
119
|
+
|
|
120
|
+
return (re.sub(extractor, "", query).strip(), facets)
|
|
121
|
+
|
|
122
|
+
def type_facet_filter(item: dict, key: str, facet_query: str):
|
|
123
|
+
entity_type: str = item[key]
|
|
124
|
+
return facet_query.lower() == entity_type
|
|
125
|
+
|
|
126
|
+
def facet_filter(item: dict, key: str, facet_query: str):
|
|
127
|
+
if entity := item.get(key):
|
|
128
|
+
name: str = entity.get("pretty_name") or entity.get("name")
|
|
129
|
+
return facet_query.lower() in name.lower() if name else False
|
|
130
|
+
return False
|
|
131
|
+
|
|
132
|
+
def facet_wrapper(query_func: Callable[[dict], bool], facet_query: str, facet_type: str) -> bool:
|
|
133
|
+
def inner(item: dict, query: str):
|
|
134
|
+
return query_func(item, query) and facet_funcs[facet_type](item, facet_type, facet_query)
|
|
135
|
+
return inner
|
|
136
|
+
|
|
137
|
+
facet_funcs = {
|
|
138
|
+
"type": type_facet_filter,
|
|
139
|
+
"release": facet_filter,
|
|
140
|
+
"group": facet_filter,
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
def index_searcher(item, query: str):
|
|
144
|
+
name: str = item["pretty_name"] or item["name"]
|
|
145
|
+
return unquote(query).lower() in name.lower() if query else True
|
|
146
|
+
|
|
147
|
+
text_query, facets = facet_extraction(query)
|
|
148
|
+
search_func = index_searcher
|
|
149
|
+
for facet, value in facets:
|
|
150
|
+
if facet in facet_funcs.keys():
|
|
151
|
+
search_func = facet_wrapper(query_func=search_func, facet_query=value, facet_type=facet)
|
|
152
|
+
|
|
153
|
+
if release_id:
|
|
154
|
+
all_releases = [ArgusRelease.get(id=release_id)]
|
|
155
|
+
else:
|
|
156
|
+
all_releases = ArgusRelease.objects().limit(None)
|
|
157
|
+
all_tests = ArgusTest.objects().limit(None)
|
|
158
|
+
all_groups = ArgusGroup.objects().limit(None)
|
|
159
|
+
if release_id:
|
|
160
|
+
all_tests = all_tests.filter(release_id=release_id)
|
|
161
|
+
all_groups = all_groups.filter(release_id=release_id)
|
|
162
|
+
release_by_id = {release.id: partial(cls.index_mapper, type="release")(release) for release in all_releases}
|
|
163
|
+
group_by_id = {group.id: partial(cls.index_mapper, type="group")(group) for group in all_groups}
|
|
164
|
+
index = [cls.index_mapper(t) for t in all_tests]
|
|
165
|
+
index = [*release_by_id.values(), *group_by_id.values(), *index]
|
|
166
|
+
for item in index:
|
|
167
|
+
item["group"] = group_by_id.get(item.get("group_id"))
|
|
168
|
+
item["release"] = release_by_id.get(item.get("release_id"))
|
|
169
|
+
|
|
170
|
+
results = filter(partial(search_func, query=text_query), index)
|
|
171
|
+
|
|
172
|
+
return [{"id": cls.ADD_ALL_ID, "name": "Add all...", "type": "special"}, *list(res for res in results if check_visibility(res))]
|
|
@@ -0,0 +1,489 @@
|
|
|
1
|
+
from collections import defaultdict
|
|
2
|
+
from datetime import datetime, timedelta
|
|
3
|
+
from functools import reduce
|
|
4
|
+
import json
|
|
5
|
+
import logging
|
|
6
|
+
import re
|
|
7
|
+
from sys import prefix
|
|
8
|
+
import time
|
|
9
|
+
from typing import Any
|
|
10
|
+
from uuid import UUID
|
|
11
|
+
|
|
12
|
+
import boto3
|
|
13
|
+
import magic
|
|
14
|
+
import requests
|
|
15
|
+
from flask import current_app, g
|
|
16
|
+
from cassandra.query import BatchStatement, ConsistencyLevel
|
|
17
|
+
from cassandra.cqlengine.query import BatchQuery
|
|
18
|
+
from argus.backend.db import ScyllaCluster
|
|
19
|
+
|
|
20
|
+
from argus.backend.models.web import (
|
|
21
|
+
ArgusEvent,
|
|
22
|
+
ArgusEventTypes,
|
|
23
|
+
ArgusNotificationSourceTypes,
|
|
24
|
+
ArgusNotificationTypes,
|
|
25
|
+
ArgusRelease,
|
|
26
|
+
ArgusTest,
|
|
27
|
+
ArgusTestRunComment,
|
|
28
|
+
User,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
from argus.backend.plugins.core import PluginInfoBase, PluginModelBase
|
|
32
|
+
|
|
33
|
+
from argus.backend.plugins.loader import AVAILABLE_PLUGINS
|
|
34
|
+
from argus.backend.events.event_processors import EVENT_PROCESSORS
|
|
35
|
+
from argus.backend.plugins.sct.testrun import SCTTestRun
|
|
36
|
+
from argus.backend.plugins.sirenada.model import SirenadaRun
|
|
37
|
+
from argus.backend.service.event_service import EventService
|
|
38
|
+
from argus.backend.service.notification_manager import NotificationManagerService
|
|
39
|
+
from argus.backend.service.stats import ComparableTestStatus
|
|
40
|
+
from argus.backend.util.common import chunk, get_build_number, strip_html_tags
|
|
41
|
+
from argus.common.enums import TestInvestigationStatus, TestStatus
|
|
42
|
+
|
|
43
|
+
LOGGER = logging.getLogger(__name__)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class TestRunServiceException(Exception):
|
|
47
|
+
pass
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class TestRunService:
|
|
51
|
+
ASSIGNEE_PLACEHOLDER = "none-none-none"
|
|
52
|
+
|
|
53
|
+
RE_MENTION = r"@[A-Za-z\d](?:[A-Za-z\d]|-(?=[A-Za-z\d])){0,38}"
|
|
54
|
+
|
|
55
|
+
plugins = AVAILABLE_PLUGINS
|
|
56
|
+
|
|
57
|
+
def __init__(self) -> None:
|
|
58
|
+
self.notification_manager = NotificationManagerService()
|
|
59
|
+
self.s3 = boto3.client(service_name="s3", aws_access_key_id=current_app.config.get(
|
|
60
|
+
"AWS_CLIENT_ID"), aws_secret_access_key=current_app.config.get("AWS_CLIENT_SECRET"))
|
|
61
|
+
|
|
62
|
+
def get_plugin(self, plugin_name: str) -> PluginInfoBase | None:
|
|
63
|
+
return self.plugins.get(plugin_name)
|
|
64
|
+
|
|
65
|
+
def get_run(self, run_type: str, run_id: UUID) -> PluginModelBase:
|
|
66
|
+
plugin = self.plugins.get(run_type)
|
|
67
|
+
if plugin:
|
|
68
|
+
try:
|
|
69
|
+
return plugin.model.get(id=run_id)
|
|
70
|
+
except plugin.model.DoesNotExist:
|
|
71
|
+
return None
|
|
72
|
+
|
|
73
|
+
def get_runs_by_test_id(self, test_id: UUID, additional_runs: list[UUID], limit: int = 10):
|
|
74
|
+
test: ArgusTest = ArgusTest.get(id=test_id)
|
|
75
|
+
plugin = self.get_plugin(plugin_name=test.plugin_name)
|
|
76
|
+
if not plugin:
|
|
77
|
+
return []
|
|
78
|
+
|
|
79
|
+
last_runs: list[dict] = plugin.model.get_run_meta_by_build_id(build_id=test.build_system_id, limit=limit)
|
|
80
|
+
last_runs_ids = [run["id"] for run in last_runs]
|
|
81
|
+
for added_run in additional_runs:
|
|
82
|
+
if added_run not in last_runs_ids:
|
|
83
|
+
last_runs.extend(plugin.model.get_run_meta_by_run_id(run_id=added_run))
|
|
84
|
+
|
|
85
|
+
for row in last_runs:
|
|
86
|
+
row["build_number"] = get_build_number(build_job_url=row["build_job_url"])
|
|
87
|
+
|
|
88
|
+
last_runs = sorted(last_runs, reverse=True, key=lambda run: (
|
|
89
|
+
run["build_number"], ComparableTestStatus(TestStatus(run["status"]))))
|
|
90
|
+
|
|
91
|
+
return last_runs
|
|
92
|
+
|
|
93
|
+
def get_runs_by_id(self, test_id: UUID, runs: list[UUID]): # FIXME: Not needed, use get_run and individual polling
|
|
94
|
+
# This is a batch request.
|
|
95
|
+
test = ArgusTest.get(id=test_id)
|
|
96
|
+
plugin = self.get_plugin(plugin_name=test.plugin_name)
|
|
97
|
+
polled_runs: list[PluginModelBase] = []
|
|
98
|
+
for run_id in runs:
|
|
99
|
+
try:
|
|
100
|
+
run: PluginModelBase = plugin.model.get(id=run_id)
|
|
101
|
+
polled_runs.append(run)
|
|
102
|
+
except plugin.model.DoesNotExist:
|
|
103
|
+
pass
|
|
104
|
+
|
|
105
|
+
response = {str(run.id): run for run in polled_runs}
|
|
106
|
+
return response
|
|
107
|
+
|
|
108
|
+
def change_run_status(self, test_id: UUID, run_id: UUID, new_status: TestStatus):
|
|
109
|
+
try:
|
|
110
|
+
test = ArgusTest.get(id=test_id)
|
|
111
|
+
except ArgusTest.DoesNotExist as exc:
|
|
112
|
+
raise TestRunServiceException("Test entity does not exist for provided test_id", test_id) from exc
|
|
113
|
+
plugin = self.get_plugin(plugin_name=test.plugin_name)
|
|
114
|
+
run: PluginModelBase = plugin.model.get(id=run_id)
|
|
115
|
+
old_status = run.status
|
|
116
|
+
run.status = new_status.value
|
|
117
|
+
run.save()
|
|
118
|
+
|
|
119
|
+
EventService.create_run_event(
|
|
120
|
+
kind=ArgusEventTypes.TestRunStatusChanged,
|
|
121
|
+
body={
|
|
122
|
+
"message": "Status was changed from {old_status} to {new_status} by {username}",
|
|
123
|
+
"old_status": old_status,
|
|
124
|
+
"new_status": new_status.value,
|
|
125
|
+
"username": g.user.username
|
|
126
|
+
},
|
|
127
|
+
user_id=g.user.id,
|
|
128
|
+
run_id=run.id,
|
|
129
|
+
release_id=test.release_id,
|
|
130
|
+
group_id=test.group_id,
|
|
131
|
+
test_id=test.id
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
return {
|
|
135
|
+
"test_run_id": run.id,
|
|
136
|
+
"status": new_status
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
@staticmethod
|
|
140
|
+
def _match_s3_link(link: str) -> re.Match:
|
|
141
|
+
return re.match(r"(https:\/\/)?(?P<bucket>[\w\-]*)\.s3(?P<region>\.[\w\-\d]*)?\.amazonaws.com\/(?P<key>.+)", link)
|
|
142
|
+
|
|
143
|
+
def get_log(self, plugin_name: str, run_id: UUID, log_name: str):
|
|
144
|
+
plugin = self.get_plugin(plugin_name=plugin_name)
|
|
145
|
+
run: PluginModelBase = plugin.model.get(id=run_id)
|
|
146
|
+
|
|
147
|
+
link = {log[0]: log[1] for log in run.logs}.get(log_name)
|
|
148
|
+
if not link:
|
|
149
|
+
raise TestRunServiceException(f"Log name {log_name} not found.")
|
|
150
|
+
match = self._match_s3_link(link)
|
|
151
|
+
if not match:
|
|
152
|
+
return link
|
|
153
|
+
presigned_url = self.s3.generate_presigned_url(ClientMethod="get_object", Params={
|
|
154
|
+
"Bucket": match.group("bucket"), "Key": match.group("key")}, ExpiresIn=3600)
|
|
155
|
+
|
|
156
|
+
return presigned_url
|
|
157
|
+
|
|
158
|
+
def resolve_artifact_size(self, link: str):
|
|
159
|
+
|
|
160
|
+
match = self._match_s3_link(link)
|
|
161
|
+
|
|
162
|
+
if not match:
|
|
163
|
+
res = requests.head(link)
|
|
164
|
+
if res.status_code != 200:
|
|
165
|
+
raise Exception("Error requesting resource")
|
|
166
|
+
|
|
167
|
+
length = res.headers.get("Content-Length")
|
|
168
|
+
if length:
|
|
169
|
+
length = int(length)
|
|
170
|
+
|
|
171
|
+
return length
|
|
172
|
+
|
|
173
|
+
obj = self.s3.get_object(Bucket=match.group("bucket"), Key=match.group("key"))
|
|
174
|
+
return obj["ContentLength"]
|
|
175
|
+
|
|
176
|
+
def proxy_stored_s3_image(self, plugin_name: str, run_id: UUID | str, image_name: str):
|
|
177
|
+
plugin = self.get_plugin(plugin_name=plugin_name)
|
|
178
|
+
run: SCTTestRun | SirenadaRun = plugin.model.get(id=run_id)
|
|
179
|
+
match run:
|
|
180
|
+
case SCTTestRun():
|
|
181
|
+
screenshot = {scr.split("/")[-1]: scr for scr in run.screenshots}.get(image_name)
|
|
182
|
+
case SirenadaRun():
|
|
183
|
+
screenshot = {
|
|
184
|
+
scr.split("/")[-1]: scr for scr in [result.screenshot_file for result in run.results]}.get(image_name)
|
|
185
|
+
|
|
186
|
+
match = self._match_s3_link(screenshot)
|
|
187
|
+
if not match:
|
|
188
|
+
return screenshot
|
|
189
|
+
|
|
190
|
+
return self.s3.generate_presigned_url(ClientMethod="get_object", Params={"Bucket": match.group("bucket"), "Key": match.group("key")}, ExpiresIn=3600)
|
|
191
|
+
|
|
192
|
+
def proxy_s3_image(self, bucket_name: str, bucket_path: str):
|
|
193
|
+
if bucket_name not in current_app.config.get("S3_ALLOWED_BUCKETS", []):
|
|
194
|
+
raise TestRunServiceException(f"{bucket_name} is not an allowed S3 bucket to pull from")
|
|
195
|
+
|
|
196
|
+
obj = self.s3.get_object(Bucket=bucket_name, Key=bucket_path)
|
|
197
|
+
header = obj["Body"].read(1024)
|
|
198
|
+
mime = magic.from_buffer(header, mime=True)
|
|
199
|
+
if "image" not in mime.lower():
|
|
200
|
+
raise TestRunServiceException(f"Cannot proxy a non-image file: {mime}", mime)
|
|
201
|
+
|
|
202
|
+
return self.s3.generate_presigned_url(ClientMethod="get_object", Params={"Bucket": bucket_name, "Key": bucket_path}, ExpiresIn=600)
|
|
203
|
+
|
|
204
|
+
def change_run_investigation_status(self, test_id: UUID, run_id: UUID, new_status: TestInvestigationStatus):
|
|
205
|
+
test = ArgusTest.get(id=test_id)
|
|
206
|
+
plugin = self.get_plugin(plugin_name=test.plugin_name)
|
|
207
|
+
run: PluginModelBase = plugin.model.get(id=run_id)
|
|
208
|
+
old_status = run.investigation_status
|
|
209
|
+
run.investigation_status = new_status.value
|
|
210
|
+
run.save()
|
|
211
|
+
|
|
212
|
+
EventService.create_run_event(
|
|
213
|
+
kind=ArgusEventTypes.TestRunStatusChanged,
|
|
214
|
+
body={
|
|
215
|
+
"message": "Investigation status was changed from {old_status} to {new_status} by {username}",
|
|
216
|
+
"old_status": old_status,
|
|
217
|
+
"new_status": new_status.value,
|
|
218
|
+
"username": g.user.username
|
|
219
|
+
},
|
|
220
|
+
user_id=g.user.id,
|
|
221
|
+
run_id=run.id,
|
|
222
|
+
release_id=test.release_id,
|
|
223
|
+
group_id=test.group_id,
|
|
224
|
+
test_id=test.id
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
return {
|
|
228
|
+
"test_run_id": run.id,
|
|
229
|
+
"investigation_status": new_status
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
def change_run_assignee(self, test_id: UUID, run_id: UUID, new_assignee: UUID | None):
|
|
233
|
+
test = ArgusTest.get(id=test_id)
|
|
234
|
+
plugin = self.get_plugin(plugin_name=test.plugin_name)
|
|
235
|
+
if not plugin:
|
|
236
|
+
return {
|
|
237
|
+
"test_run_id": run.id,
|
|
238
|
+
"assignee": None
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
run: PluginModelBase = plugin.model.get(id=run_id)
|
|
242
|
+
old_assignee = run.assignee
|
|
243
|
+
run.assignee = new_assignee
|
|
244
|
+
run.save()
|
|
245
|
+
|
|
246
|
+
if new_assignee:
|
|
247
|
+
new_assignee_user = User.get(id=new_assignee)
|
|
248
|
+
else:
|
|
249
|
+
new_assignee_user = None
|
|
250
|
+
if old_assignee:
|
|
251
|
+
try:
|
|
252
|
+
old_assignee_user = User.get(id=old_assignee)
|
|
253
|
+
except User.DoesNotExist:
|
|
254
|
+
LOGGER.warning("Non existent assignee was present on the run %s for test %s: %s",
|
|
255
|
+
run_id, test_id, old_assignee)
|
|
256
|
+
old_assignee = None
|
|
257
|
+
EventService.create_run_event(
|
|
258
|
+
kind=ArgusEventTypes.AssigneeChanged,
|
|
259
|
+
body={
|
|
260
|
+
"message": "Assignee was changed from \"{old_user}\" to \"{new_user}\" by {username}",
|
|
261
|
+
"old_user": old_assignee_user.username if old_assignee else "None",
|
|
262
|
+
"new_user": new_assignee_user.username if new_assignee else "None",
|
|
263
|
+
"username": g.user.username
|
|
264
|
+
},
|
|
265
|
+
user_id=g.user.id,
|
|
266
|
+
run_id=run.id,
|
|
267
|
+
release_id=test.release_id,
|
|
268
|
+
group_id=test.group_id,
|
|
269
|
+
test_id=test.id
|
|
270
|
+
)
|
|
271
|
+
if new_assignee_user.id != g.user.id:
|
|
272
|
+
self.notification_manager.send_notification(
|
|
273
|
+
receiver=new_assignee_user.id,
|
|
274
|
+
sender=g.user.id,
|
|
275
|
+
notification_type=ArgusNotificationTypes.AssigneeChange,
|
|
276
|
+
source_type=ArgusNotificationSourceTypes.TestRun,
|
|
277
|
+
source_id=run.id,
|
|
278
|
+
source_message=str(run.test_id),
|
|
279
|
+
content_params={
|
|
280
|
+
"username": g.user.username,
|
|
281
|
+
"run_id": run.id,
|
|
282
|
+
"test_id": test.id,
|
|
283
|
+
"build_id": run.build_id,
|
|
284
|
+
"build_number": get_build_number(run.build_job_url),
|
|
285
|
+
}
|
|
286
|
+
)
|
|
287
|
+
return {
|
|
288
|
+
"test_run_id": run.id,
|
|
289
|
+
"assignee": str(new_assignee_user.id) if new_assignee_user else None
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
def get_run_comment(self, comment_id: UUID):
|
|
293
|
+
try:
|
|
294
|
+
return ArgusTestRunComment.get(id=comment_id)
|
|
295
|
+
except ArgusTestRunComment.DoesNotExist:
|
|
296
|
+
return None
|
|
297
|
+
|
|
298
|
+
def get_run_comments(self, run_id: UUID):
|
|
299
|
+
return sorted(ArgusTestRunComment.filter(test_run_id=run_id).all(), key=lambda c: c.posted_at)
|
|
300
|
+
|
|
301
|
+
def post_run_comment(self, test_id: UUID, run_id: UUID, message: str, reactions: dict, mentions: list[str]):
|
|
302
|
+
message_stripped = strip_html_tags(message)
|
|
303
|
+
|
|
304
|
+
mentions = set(mentions)
|
|
305
|
+
for potential_mention in re.findall(self.RE_MENTION, message_stripped):
|
|
306
|
+
if user := User.exists_by_name(potential_mention.lstrip("@")):
|
|
307
|
+
mentions.add(user) if user.id != g.user.id else None
|
|
308
|
+
|
|
309
|
+
test: ArgusTest = ArgusTest.get(id=test_id)
|
|
310
|
+
plugin = self.get_plugin(test.plugin_name)
|
|
311
|
+
release: ArgusRelease = ArgusRelease.get(id=test.release_id)
|
|
312
|
+
comment = ArgusTestRunComment()
|
|
313
|
+
comment.test_id = test.id
|
|
314
|
+
comment.message = message_stripped
|
|
315
|
+
comment.reactions = reactions
|
|
316
|
+
comment.mentions = [m.id for m in mentions]
|
|
317
|
+
comment.test_run_id = run_id
|
|
318
|
+
comment.release_id = release.id
|
|
319
|
+
comment.user_id = g.user.id
|
|
320
|
+
comment.posted_at = time.time()
|
|
321
|
+
comment.save()
|
|
322
|
+
|
|
323
|
+
run: PluginModelBase = plugin.model.get(id=run_id)
|
|
324
|
+
build_number = get_build_number(build_job_url=run.build_job_url)
|
|
325
|
+
for mention in mentions:
|
|
326
|
+
params = {
|
|
327
|
+
"username": g.user.username,
|
|
328
|
+
"run_id": comment.test_run_id,
|
|
329
|
+
"test_id": test.id,
|
|
330
|
+
"build_id": run.build_id,
|
|
331
|
+
"build_number": build_number,
|
|
332
|
+
}
|
|
333
|
+
self.notification_manager.send_notification(
|
|
334
|
+
receiver=mention.id,
|
|
335
|
+
sender=comment.user_id,
|
|
336
|
+
notification_type=ArgusNotificationTypes.Mention,
|
|
337
|
+
source_type=ArgusNotificationSourceTypes.Comment,
|
|
338
|
+
source_id=comment.id,
|
|
339
|
+
source_message=comment.message,
|
|
340
|
+
content_params=params
|
|
341
|
+
)
|
|
342
|
+
|
|
343
|
+
EventService.create_run_event(kind=ArgusEventTypes.TestRunCommentPosted, body={
|
|
344
|
+
"message": "A comment was posted by {username}",
|
|
345
|
+
"username": g.user.username
|
|
346
|
+
}, user_id=g.user.id, run_id=run_id, release_id=release.id, test_id=test.id)
|
|
347
|
+
|
|
348
|
+
return self.get_run_comments(run_id=run_id)
|
|
349
|
+
|
|
350
|
+
def delete_run_comment(self, comment_id: UUID, test_id: UUID, run_id: UUID):
|
|
351
|
+
comment: ArgusTestRunComment = ArgusTestRunComment.get(id=comment_id)
|
|
352
|
+
if comment.user_id != g.user.id:
|
|
353
|
+
raise Exception("Unable to delete other user comments")
|
|
354
|
+
comment.delete()
|
|
355
|
+
|
|
356
|
+
EventService.create_run_event(kind=ArgusEventTypes.TestRunCommentDeleted, body={
|
|
357
|
+
"message": "A comment was deleted by {username}",
|
|
358
|
+
"username": g.user.username
|
|
359
|
+
}, user_id=g.user.id, run_id=run_id, release_id=comment.release_id, test_id=test_id)
|
|
360
|
+
|
|
361
|
+
return self.get_run_comments(run_id=run_id)
|
|
362
|
+
|
|
363
|
+
def update_run_comment(self, comment_id: UUID, test_id: UUID, run_id: UUID, message: str, mentions: list[str], reactions: dict):
|
|
364
|
+
comment: ArgusTestRunComment = ArgusTestRunComment.get(id=comment_id)
|
|
365
|
+
if comment.user_id != g.user.id:
|
|
366
|
+
raise Exception("Unable to edit other user comments")
|
|
367
|
+
comment.message = strip_html_tags(message)
|
|
368
|
+
comment.reactions = reactions
|
|
369
|
+
comment.mentions = mentions
|
|
370
|
+
comment.save()
|
|
371
|
+
|
|
372
|
+
EventService.create_run_event(kind=ArgusEventTypes.TestRunCommentUpdated, body={
|
|
373
|
+
"message": "A comment was edited by {username}",
|
|
374
|
+
"username": g.user.username
|
|
375
|
+
}, user_id=g.user.id, run_id=run_id, release_id=comment.release_id, test_id=test_id)
|
|
376
|
+
|
|
377
|
+
return self.get_run_comments(run_id=run_id)
|
|
378
|
+
|
|
379
|
+
def get_run_events(self, run_id: UUID):
|
|
380
|
+
response = {}
|
|
381
|
+
all_events = ArgusEvent.filter(run_id=run_id).all()
|
|
382
|
+
all_events = sorted(all_events, key=lambda ev: ev.created_at)
|
|
383
|
+
response["run_id"] = run_id
|
|
384
|
+
response["raw_events"] = [dict(event.items()) for event in all_events]
|
|
385
|
+
response["events"] = {
|
|
386
|
+
str(event.id): EVENT_PROCESSORS.get(event.kind)(json.loads(event.body))
|
|
387
|
+
for event in all_events
|
|
388
|
+
}
|
|
389
|
+
return response
|
|
390
|
+
|
|
391
|
+
def resolve_run_build_id_and_number_multiple(self, runs: list[tuple[UUID, UUID]]) -> dict[UUID, dict[str, Any]]:
|
|
392
|
+
test_ids = [r[0] for r in runs]
|
|
393
|
+
all_tests: list = []
|
|
394
|
+
for id_slice in chunk(test_ids):
|
|
395
|
+
all_tests.extend(ArgusTest.filter(id__in=id_slice).all())
|
|
396
|
+
|
|
397
|
+
tests: dict[str, ArgusTest] = {str(t.id): t for t in all_tests}
|
|
398
|
+
runs_by_plugin = reduce(lambda acc, val: acc[tests[val[0]].plugin_name].append(
|
|
399
|
+
val[1]) or acc, runs, defaultdict(list))
|
|
400
|
+
all_runs = {}
|
|
401
|
+
for plugin, run_ids in runs_by_plugin.items():
|
|
402
|
+
model = AVAILABLE_PLUGINS.get(plugin).model
|
|
403
|
+
model_runs = []
|
|
404
|
+
for run_id in run_ids:
|
|
405
|
+
model_runs.append(model.filter(id=run_id).only(
|
|
406
|
+
["build_id", "start_time", "build_job_url", "id", "test_id"]).get())
|
|
407
|
+
all_runs.update(
|
|
408
|
+
{str(run["id"]): {**run, "build_number": get_build_number(run["build_job_url"])} for run in model_runs})
|
|
409
|
+
|
|
410
|
+
return all_runs
|
|
411
|
+
|
|
412
|
+
def terminate_stuck_runs(self):
|
|
413
|
+
sct = AVAILABLE_PLUGINS.get("scylla-cluster-tests").model
|
|
414
|
+
now = datetime.utcnow()
|
|
415
|
+
stuck_period = now - timedelta(minutes=45)
|
|
416
|
+
stuck_runs_running = sct.filter(heartbeat__lt=int(
|
|
417
|
+
stuck_period.timestamp()), status=TestStatus.RUNNING.value).allow_filtering().all()
|
|
418
|
+
stuck_runs_created = sct.filter(heartbeat__lt=int(
|
|
419
|
+
stuck_period.timestamp()), status=TestStatus.CREATED.value).allow_filtering().all()
|
|
420
|
+
|
|
421
|
+
all_stuck_runs = [*stuck_runs_running, *stuck_runs_created]
|
|
422
|
+
LOGGER.info("Found %s stuck runs", len(all_stuck_runs))
|
|
423
|
+
|
|
424
|
+
for run in all_stuck_runs:
|
|
425
|
+
LOGGER.info("Will set %s as ABORTED", run.id)
|
|
426
|
+
old_status = run.status
|
|
427
|
+
run.status = TestStatus.ABORTED.value
|
|
428
|
+
run.save()
|
|
429
|
+
|
|
430
|
+
EventService.create_run_event(
|
|
431
|
+
kind=ArgusEventTypes.TestRunStatusChanged,
|
|
432
|
+
body={
|
|
433
|
+
"message": "Run was automatically terminated due to not responding for more than 45 minutes "
|
|
434
|
+
"(Status changed from {old_status} to {new_status}) by {username}",
|
|
435
|
+
"old_status": old_status,
|
|
436
|
+
"new_status": run.status,
|
|
437
|
+
"username": g.user.username
|
|
438
|
+
},
|
|
439
|
+
user_id=g.user.id,
|
|
440
|
+
run_id=run.id,
|
|
441
|
+
release_id=run.release_id,
|
|
442
|
+
group_id=run.group_id,
|
|
443
|
+
test_id=run.test_id
|
|
444
|
+
)
|
|
445
|
+
|
|
446
|
+
return len(all_stuck_runs)
|
|
447
|
+
|
|
448
|
+
def ignore_jobs(self, test_id: UUID, reason: str):
|
|
449
|
+
test: ArgusTest = ArgusTest.get(id=test_id)
|
|
450
|
+
plugin = self.get_plugin(plugin_name=test.plugin_name)
|
|
451
|
+
|
|
452
|
+
if not reason:
|
|
453
|
+
raise TestRunServiceException("Reason for ignore cannot be empty")
|
|
454
|
+
|
|
455
|
+
cluster = ScyllaCluster.get()
|
|
456
|
+
batch = BatchStatement(consistency_level=ConsistencyLevel.QUORUM)
|
|
457
|
+
event_batch = BatchQuery()
|
|
458
|
+
jobs_affected = 0
|
|
459
|
+
for job in plugin.model.get_jobs_meta_by_test_id(test.id):
|
|
460
|
+
if job["status"] != TestStatus.PASSED and job["investigation_status"] == TestInvestigationStatus.NOT_INVESTIGATED:
|
|
461
|
+
batch.add(
|
|
462
|
+
plugin.model.prepare_investigation_status_update_query(
|
|
463
|
+
build_id=job["build_id"],
|
|
464
|
+
start_time=job["start_time"],
|
|
465
|
+
new_status=TestInvestigationStatus.IGNORED
|
|
466
|
+
)
|
|
467
|
+
)
|
|
468
|
+
|
|
469
|
+
ArgusEvent.batch(event_batch).create(
|
|
470
|
+
release_id=job["release_id"],
|
|
471
|
+
group_id=job["group_id"],
|
|
472
|
+
test_id=test_id,
|
|
473
|
+
user_id=g.user.id,
|
|
474
|
+
run_id=job["id"],
|
|
475
|
+
body=json.dumps({
|
|
476
|
+
"message": "Run was marked as ignored by {username} due to the following reason: {reason}",
|
|
477
|
+
"username": g.user.username,
|
|
478
|
+
"reason": reason,
|
|
479
|
+
}, ensure_ascii=True, separators=(',', ':')),
|
|
480
|
+
kind=ArgusEventTypes.TestRunBatchInvestigationStatusChange.value,
|
|
481
|
+
created_at=datetime.utcnow(),
|
|
482
|
+
)
|
|
483
|
+
|
|
484
|
+
jobs_affected += 1
|
|
485
|
+
|
|
486
|
+
cluster.session.execute(batch)
|
|
487
|
+
event_batch.execute()
|
|
488
|
+
|
|
489
|
+
return jobs_affected
|