argus-alm 0.12.2__py3-none-any.whl → 0.12.4b1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- argus/backend/cli.py +1 -1
- argus/backend/controller/admin_api.py +26 -0
- argus/backend/controller/api.py +26 -1
- argus/backend/controller/main.py +21 -0
- argus/backend/controller/testrun_api.py +132 -1
- argus/backend/controller/view_api.py +162 -0
- argus/backend/models/web.py +16 -0
- argus/backend/plugins/core.py +28 -5
- argus/backend/plugins/driver_matrix_tests/controller.py +39 -0
- argus/backend/plugins/driver_matrix_tests/model.py +252 -4
- argus/backend/plugins/driver_matrix_tests/raw_types.py +27 -0
- argus/backend/plugins/driver_matrix_tests/service.py +18 -0
- argus/backend/plugins/driver_matrix_tests/udt.py +14 -13
- argus/backend/plugins/generic/model.py +6 -3
- argus/backend/plugins/loader.py +2 -2
- argus/backend/plugins/sct/controller.py +31 -0
- argus/backend/plugins/sct/plugin.py +2 -1
- argus/backend/plugins/sct/service.py +101 -3
- argus/backend/plugins/sct/testrun.py +8 -2
- argus/backend/plugins/sct/types.py +18 -0
- argus/backend/plugins/sct/udt.py +6 -0
- argus/backend/plugins/sirenada/model.py +1 -1
- argus/backend/service/argus_service.py +116 -11
- argus/backend/service/build_system_monitor.py +37 -7
- argus/backend/service/jenkins_service.py +176 -1
- argus/backend/service/release_manager.py +14 -0
- argus/backend/service/stats.py +179 -21
- argus/backend/service/testrun.py +44 -5
- argus/backend/service/views.py +258 -0
- argus/backend/template_filters.py +7 -0
- argus/backend/util/common.py +14 -2
- argus/client/driver_matrix_tests/cli.py +110 -0
- argus/client/driver_matrix_tests/client.py +56 -193
- argus/client/sct/client.py +34 -0
- argus_alm-0.12.4b1.dist-info/METADATA +129 -0
- {argus_alm-0.12.2.dist-info → argus_alm-0.12.4b1.dist-info}/RECORD +39 -36
- {argus_alm-0.12.2.dist-info → argus_alm-0.12.4b1.dist-info}/WHEEL +1 -1
- {argus_alm-0.12.2.dist-info → argus_alm-0.12.4b1.dist-info}/entry_points.txt +1 -0
- argus_alm-0.12.2.dist-info/METADATA +0 -206
- {argus_alm-0.12.2.dist-info → argus_alm-0.12.4b1.dist-info}/LICENSE +0 -0
|
@@ -6,7 +6,7 @@ from typing import Optional
|
|
|
6
6
|
from uuid import UUID
|
|
7
7
|
|
|
8
8
|
from cassandra.cqlengine import columns
|
|
9
|
-
from cassandra.cqlengine.models import _DoesNotExist
|
|
9
|
+
from cassandra.cqlengine.models import _DoesNotExist, Model
|
|
10
10
|
from argus.backend.db import ScyllaCluster
|
|
11
11
|
from argus.backend.models.web import ArgusRelease
|
|
12
12
|
from argus.backend.plugins.core import PluginModelBase
|
|
@@ -113,7 +113,7 @@ class SCTTestRun(PluginModelBase):
|
|
|
113
113
|
@classmethod
|
|
114
114
|
def _stats_query(cls) -> str:
|
|
115
115
|
return ("SELECT id, test_id, group_id, release_id, status, start_time, build_job_url, build_id, "
|
|
116
|
-
f"assignee, end_time, investigation_status, heartbeat, scylla_version FROM {cls.table_name()} WHERE
|
|
116
|
+
f"assignee, end_time, investigation_status, heartbeat, scylla_version FROM {cls.table_name()} WHERE build_id IN ? PER PARTITION LIMIT 15")
|
|
117
117
|
|
|
118
118
|
@classmethod
|
|
119
119
|
def load_test_run(cls, run_id: UUID) -> 'SCTTestRun':
|
|
@@ -248,3 +248,9 @@ class SCTTestRun(PluginModelBase):
|
|
|
248
248
|
self._add_new_event_type(event)
|
|
249
249
|
|
|
250
250
|
self._collect_event_message(event, event_message)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
class SCTJunitReports(Model):
|
|
254
|
+
test_id = columns.UUID(primary_key=True, partition_key=True, required=True)
|
|
255
|
+
file_name = columns.Text(primary_key=True, required=True)
|
|
256
|
+
report = columns.Text(required=True)
|
|
@@ -36,3 +36,21 @@ class PerformanceResultsRequest(TypedDict):
|
|
|
36
36
|
perf_total_errors: str
|
|
37
37
|
|
|
38
38
|
histograms: list[dict[str, RawHDRHistogram]] | None
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class InstanceInfoUpdateRequest(TypedDict):
|
|
42
|
+
provider: str
|
|
43
|
+
region: str
|
|
44
|
+
public_ip: str
|
|
45
|
+
private_ip: str
|
|
46
|
+
dc_name: str
|
|
47
|
+
rack_name: str
|
|
48
|
+
creation_time: int
|
|
49
|
+
termination_time: int
|
|
50
|
+
termination_reason: str
|
|
51
|
+
shards_amount: int
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class ResourceUpdateRequest(TypedDict):
|
|
55
|
+
state: str
|
|
56
|
+
instance_info: InstanceInfoUpdateRequest
|
argus/backend/plugins/sct/udt.py
CHANGED
|
@@ -14,6 +14,12 @@ class PackageVersion(UserType):
|
|
|
14
14
|
build_id = columns.Text()
|
|
15
15
|
|
|
16
16
|
|
|
17
|
+
def __eq__(self, other):
|
|
18
|
+
if isinstance(other, PackageVersion):
|
|
19
|
+
return all(getattr(self, a) == getattr(other, a) for a in ["name", "version", "date", "revision_id", "build_id"])
|
|
20
|
+
return super().__eq__(other)
|
|
21
|
+
|
|
22
|
+
|
|
17
23
|
class CloudInstanceDetails(UserType):
|
|
18
24
|
__type_name__ = "CloudInstanceDetails_v3"
|
|
19
25
|
provider = columns.Text()
|
|
@@ -47,7 +47,7 @@ class SirenadaRun(PluginModelBase):
|
|
|
47
47
|
@classmethod
|
|
48
48
|
def _stats_query(cls) -> str:
|
|
49
49
|
return ("SELECT id, test_id, group_id, release_id, status, start_time, build_job_url, build_id, "
|
|
50
|
-
f"assignee, end_time, investigation_status, heartbeat, scylla_version FROM {cls.table_name()} WHERE
|
|
50
|
+
f"assignee, end_time, investigation_status, heartbeat, scylla_version FROM {cls.table_name()} WHERE build_id IN ? PER PARTITION LIMIT 15")
|
|
51
51
|
|
|
52
52
|
@classmethod
|
|
53
53
|
def get_distinct_product_versions(cls, release: ArgusRelease, cluster: ScyllaCluster = None) -> list[str]:
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
from math import ceil
|
|
2
|
+
from dataclasses import dataclass
|
|
1
3
|
import subprocess
|
|
2
4
|
import json
|
|
3
5
|
import logging
|
|
@@ -25,9 +27,20 @@ from argus.backend.models.web import (
|
|
|
25
27
|
)
|
|
26
28
|
from argus.backend.events.event_processors import EVENT_PROCESSORS
|
|
27
29
|
from argus.backend.service.testrun import TestRunService
|
|
30
|
+
from argus.backend.util.common import chunk
|
|
28
31
|
|
|
29
32
|
LOGGER = logging.getLogger(__name__)
|
|
30
33
|
|
|
34
|
+
@dataclass(init=True, frozen=True)
|
|
35
|
+
class ScheduleUpdateRequest:
|
|
36
|
+
release_id: UUID
|
|
37
|
+
schedule_id: UUID
|
|
38
|
+
assignee: UUID
|
|
39
|
+
new_tests: list[UUID]
|
|
40
|
+
old_tests: list[UUID]
|
|
41
|
+
comments: dict[UUID, str]
|
|
42
|
+
|
|
43
|
+
|
|
31
44
|
|
|
32
45
|
class ArgusService:
|
|
33
46
|
# pylint: disable=no-self-use,too-many-arguments,too-many-instance-attributes,too-many-locals, too-many-public-methods
|
|
@@ -233,7 +246,7 @@ class ArgusService:
|
|
|
233
246
|
row.save()
|
|
234
247
|
|
|
235
248
|
def submit_new_schedule(self, release: str | UUID, start_time: str, end_time: str, tests: list[str | UUID],
|
|
236
|
-
groups: list[str | UUID], assignees: list[str | UUID], tag: str) -> dict:
|
|
249
|
+
groups: list[str | UUID], assignees: list[str | UUID], tag: str, comments: dict[str, str] | None, group_ids: dict[str, str] | None) -> dict:
|
|
237
250
|
release = UUID(release) if isinstance(release, str) else release
|
|
238
251
|
if len(assignees) == 0:
|
|
239
252
|
raise Exception("Assignees not specified in the new schedule")
|
|
@@ -283,6 +296,19 @@ class ArgusService:
|
|
|
283
296
|
assignee_entity.save()
|
|
284
297
|
response["assignees"].append(assignee_id)
|
|
285
298
|
|
|
299
|
+
if comments:
|
|
300
|
+
for test_id, new_comment in comments.items():
|
|
301
|
+
try:
|
|
302
|
+
comment = ReleasePlannerComment.get(release=release, group=group_ids[test_id], test=test_id)
|
|
303
|
+
except ReleasePlannerComment.DoesNotExist:
|
|
304
|
+
comment = ReleasePlannerComment()
|
|
305
|
+
comment.release = release
|
|
306
|
+
comment.group = group_ids[test_id]
|
|
307
|
+
comment.test = test_id
|
|
308
|
+
|
|
309
|
+
comment.comment = new_comment
|
|
310
|
+
comment.save()
|
|
311
|
+
|
|
286
312
|
return response
|
|
287
313
|
|
|
288
314
|
def get_schedules_for_release(self, release_id: str | UUID) -> dict:
|
|
@@ -397,11 +423,52 @@ class ArgusService:
|
|
|
397
423
|
"newComment": new_comment,
|
|
398
424
|
}
|
|
399
425
|
|
|
426
|
+
def update_schedule(self, release_id: UUID | str, schedule_id: UUID | str, old_tests: list[UUID | str], new_tests: list[UUID | str], comments: dict[str, str], assignee: UUID | str):
|
|
427
|
+
schedule: ArgusSchedule = ArgusSchedule.get(release_id=release_id, id=schedule_id)
|
|
428
|
+
new_tests: set[UUID] = {UUID(id) for id in new_tests}
|
|
429
|
+
old_tests: set[UUID] = {UUID(id) for id in old_tests}
|
|
430
|
+
|
|
431
|
+
all_test_ids = old_tests.union(new_tests)
|
|
432
|
+
tests = []
|
|
433
|
+
for batch in chunk(all_test_ids):
|
|
434
|
+
tests.extend(ArgusTest.filter(id__in=batch).all())
|
|
435
|
+
tests_by_id: dict[UUID, ArgusTest] = { test.id: test for test in tests }
|
|
436
|
+
|
|
437
|
+
all_scheduled_tests: list[ArgusScheduleTest] = list(ArgusScheduleTest.filter(schedule_id=schedule_id).all())
|
|
438
|
+
tests_to_remove = all_test_ids.difference(new_tests)
|
|
439
|
+
for scheduled_test in all_scheduled_tests:
|
|
440
|
+
if scheduled_test.test_id in tests_to_remove:
|
|
441
|
+
test = tests_by_id.get(scheduled_test.test_id)
|
|
442
|
+
scheduled_test.delete()
|
|
443
|
+
if test:
|
|
444
|
+
self.update_schedule_comment({"newComment": "", "releaseId": test.release_id, "groupId": test.group_id, "testId": test.id})
|
|
445
|
+
|
|
446
|
+
tests_to_add = new_tests.difference(old_tests)
|
|
447
|
+
for test_id in tests_to_add:
|
|
448
|
+
entity = ArgusScheduleTest()
|
|
449
|
+
entity.id = uuid_from_time(schedule.period_start)
|
|
450
|
+
entity.schedule_id = schedule.id
|
|
451
|
+
entity.test_id = UUID(test_id) if isinstance(test_id, str) else test_id
|
|
452
|
+
entity.release_id = release_id
|
|
453
|
+
entity.save()
|
|
454
|
+
self.assign_runs_for_scheduled_test(schedule, entity.test_id, assignee)
|
|
455
|
+
|
|
456
|
+
for test_id, comment in comments.items():
|
|
457
|
+
test = tests_by_id.get(UUID(test_id))
|
|
458
|
+
if test:
|
|
459
|
+
self.update_schedule_comment({"newComment": comment, "releaseId": test.release_id, "groupId": test.group_id, "testId": test.id})
|
|
460
|
+
|
|
461
|
+
schedule_assignee: ArgusScheduleAssignee = ArgusScheduleAssignee.get(schedule_id=schedule_id)
|
|
462
|
+
schedule_assignee.assignee = assignee
|
|
463
|
+
schedule_assignee.save()
|
|
464
|
+
return True
|
|
465
|
+
|
|
400
466
|
def delete_schedule(self, payload: dict) -> dict:
|
|
401
467
|
"""
|
|
402
468
|
{
|
|
403
469
|
"release": hex-uuid,
|
|
404
|
-
"schedule_id": uuid1
|
|
470
|
+
"schedule_id": uuid1,
|
|
471
|
+
"deleteComments": bool
|
|
405
472
|
}
|
|
406
473
|
"""
|
|
407
474
|
release_id = payload.get("releaseId")
|
|
@@ -412,6 +479,8 @@ class ArgusService:
|
|
|
412
479
|
if not schedule_id:
|
|
413
480
|
raise Exception("Schedule id not specified in the request")
|
|
414
481
|
|
|
482
|
+
delete_comments = payload.get("deleteComments", False)
|
|
483
|
+
|
|
415
484
|
release = ArgusRelease.get(id=release_id)
|
|
416
485
|
schedule = ArgusSchedule.get(release_id=release.id, id=schedule_id)
|
|
417
486
|
tests = ArgusScheduleTest.filter(schedule_id=schedule.id).all()
|
|
@@ -441,6 +510,15 @@ class ArgusService:
|
|
|
441
510
|
entity.delete()
|
|
442
511
|
|
|
443
512
|
schedule.delete()
|
|
513
|
+
|
|
514
|
+
if delete_comments:
|
|
515
|
+
tests = []
|
|
516
|
+
for batch in chunk(full_schedule["tests"]):
|
|
517
|
+
tests.extend(ArgusTest.filter(id__in=batch).all())
|
|
518
|
+
|
|
519
|
+
for test in tests:
|
|
520
|
+
self.update_schedule_comment({"newComment": "", "releaseId": test.release_id, "groupId": test.group_id, "testId": test.id})
|
|
521
|
+
|
|
444
522
|
return {
|
|
445
523
|
"releaseId": release.id,
|
|
446
524
|
"scheduleId": schedule_id,
|
|
@@ -480,6 +558,15 @@ class ArgusService:
|
|
|
480
558
|
|
|
481
559
|
return response
|
|
482
560
|
|
|
561
|
+
def _batch_get_schedules_from_ids(self, release_id: UUID, schedule_ids: list[UUID]) -> list[ArgusSchedule]:
|
|
562
|
+
schedules = []
|
|
563
|
+
step_size = 90
|
|
564
|
+
for step in range(0, ceil(len(schedule_ids) / step_size)):
|
|
565
|
+
start_pos = step*step_size
|
|
566
|
+
next_slice = schedule_ids[start_pos:start_pos+step_size]
|
|
567
|
+
schedules.extend(ArgusSchedule.filter(release_id=release_id, id__in=next_slice).all())
|
|
568
|
+
return schedules
|
|
569
|
+
|
|
483
570
|
def get_groups_assignees(self, release_id: UUID | str):
|
|
484
571
|
release_id = UUID(release_id) if isinstance(release_id, str) else release_id
|
|
485
572
|
release = ArgusRelease.get(id=release_id)
|
|
@@ -487,10 +574,19 @@ class ArgusService:
|
|
|
487
574
|
groups = ArgusGroup.filter(release_id=release_id).all()
|
|
488
575
|
group_ids = [group.id for group in groups if group.enabled]
|
|
489
576
|
|
|
490
|
-
|
|
491
|
-
|
|
577
|
+
schedule_ids = set()
|
|
578
|
+
group_schedules =[]
|
|
579
|
+
step_size = 90
|
|
492
580
|
|
|
493
|
-
|
|
581
|
+
for step in range(0, ceil(len(group_ids) / step_size)):
|
|
582
|
+
start_pos = step*step_size
|
|
583
|
+
next_slice = group_ids[start_pos:start_pos+step_size]
|
|
584
|
+
group_batch = list(ArgusScheduleGroup.filter(release_id=release.id, group_id__in=next_slice).all())
|
|
585
|
+
group_schedules.extend(group_batch)
|
|
586
|
+
batch_ids = {schedule.schedule_id for schedule in group_batch}
|
|
587
|
+
schedule_ids = schedule_ids.union(batch_ids)
|
|
588
|
+
|
|
589
|
+
schedules = self._batch_get_schedules_from_ids(release.id, list(schedule_ids))
|
|
494
590
|
|
|
495
591
|
valid_schedules = schedules
|
|
496
592
|
if release.perpetual:
|
|
@@ -501,7 +597,7 @@ class ArgusService:
|
|
|
501
597
|
for schedule in valid_schedules:
|
|
502
598
|
assignees = ArgusScheduleAssignee.filter(schedule_id=schedule.id).all()
|
|
503
599
|
assignees_uuids = [assignee.assignee for assignee in assignees]
|
|
504
|
-
schedule_groups = filter(lambda g: g.schedule_id == schedule.id,
|
|
600
|
+
schedule_groups = filter(lambda g: g.schedule_id == schedule.id, group_schedules)
|
|
505
601
|
groups = {str(group.group_id): assignees_uuids for group in schedule_groups}
|
|
506
602
|
response = {**groups, **response}
|
|
507
603
|
|
|
@@ -516,10 +612,19 @@ class ArgusService:
|
|
|
516
612
|
|
|
517
613
|
test_ids = [test.id for test in tests if test.enabled]
|
|
518
614
|
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
615
|
+
schedule_ids = set()
|
|
616
|
+
test_schedules = []
|
|
617
|
+
step_size = 90
|
|
618
|
+
|
|
619
|
+
for step in range(0, ceil(len(test_ids) / step_size)):
|
|
620
|
+
start_pos = step*step_size
|
|
621
|
+
next_slice = test_ids[start_pos:start_pos+step_size]
|
|
622
|
+
test_batch = ArgusScheduleTest.filter(release_id=release.id, test_id__in=next_slice).all()
|
|
623
|
+
test_schedules.extend(test_batch)
|
|
624
|
+
batch_ids = {schedule.schedule_id for schedule in test_batch}
|
|
625
|
+
schedule_ids = schedule_ids.union(batch_ids)
|
|
626
|
+
|
|
627
|
+
schedules = self._batch_get_schedules_from_ids(release.id, list(schedule_ids))
|
|
523
628
|
|
|
524
629
|
if release.perpetual:
|
|
525
630
|
today = datetime.datetime.utcnow()
|
|
@@ -529,7 +634,7 @@ class ArgusService:
|
|
|
529
634
|
for schedule in schedules:
|
|
530
635
|
assignees = ArgusScheduleAssignee.filter(schedule_id=schedule.id).all()
|
|
531
636
|
assignees_uuids = [assignee.assignee for assignee in assignees]
|
|
532
|
-
schedule_tests = filter(lambda t: t.schedule_id == schedule.id,
|
|
637
|
+
schedule_tests = filter(lambda t: t.schedule_id == schedule.id, test_schedules)
|
|
533
638
|
tests = {str(test.test_id): assignees_uuids for test in schedule_tests}
|
|
534
639
|
response = {**tests, **response}
|
|
535
640
|
|
|
@@ -2,23 +2,28 @@ import logging
|
|
|
2
2
|
from abc import ABC, abstractmethod
|
|
3
3
|
import jenkins
|
|
4
4
|
import click
|
|
5
|
+
import re
|
|
5
6
|
from flask import current_app
|
|
6
7
|
from flask.cli import with_appcontext
|
|
7
8
|
|
|
8
9
|
from argus.backend.db import ScyllaCluster
|
|
9
|
-
from argus.backend.models.web import ArgusRelease, ArgusGroup, ArgusTest
|
|
10
|
+
from argus.backend.models.web import ArgusRelease, ArgusGroup, ArgusTest, ArgusTestException
|
|
10
11
|
from argus.backend.service.release_manager import ReleaseManagerService
|
|
11
12
|
|
|
12
13
|
LOGGER = logging.getLogger(__name__)
|
|
13
14
|
|
|
14
15
|
|
|
15
16
|
class ArgusTestsMonitor(ABC):
|
|
17
|
+
BUILD_SYSTEM_FILTERED_PREFIXES = [
|
|
18
|
+
|
|
19
|
+
]
|
|
20
|
+
|
|
16
21
|
def __init__(self) -> None:
|
|
17
22
|
self._cluster = ScyllaCluster.get()
|
|
18
23
|
self._existing_releases = list(ArgusRelease.all())
|
|
19
24
|
self._existing_groups = list(ArgusGroup.all())
|
|
20
25
|
self._existing_tests = list(ArgusTest.all())
|
|
21
|
-
self._filtered_groups: list[str] =
|
|
26
|
+
self._filtered_groups: list[str] = self.BUILD_SYSTEM_FILTERED_PREFIXES
|
|
22
27
|
|
|
23
28
|
def create_release(self, release_name):
|
|
24
29
|
# pylint: disable=no-self-use
|
|
@@ -68,17 +73,39 @@ class ArgusTestsMonitor(ABC):
|
|
|
68
73
|
|
|
69
74
|
|
|
70
75
|
class JenkinsMonitor(ArgusTestsMonitor):
|
|
76
|
+
|
|
77
|
+
BUILD_SYSTEM_FILTERED_PREFIXES = [
|
|
78
|
+
"releng",
|
|
79
|
+
]
|
|
80
|
+
|
|
81
|
+
JENKINS_MONITORED_RELEASES = [
|
|
82
|
+
r"^scylla-master$",
|
|
83
|
+
r"^scylla-staging$",
|
|
84
|
+
r"^scylla-\d+\.\d+$",
|
|
85
|
+
r"^manager-3.\d+$",
|
|
86
|
+
r"^scylla-operator/operator-master$",
|
|
87
|
+
r"^scylla-operator/operator-\d+.\d+$",
|
|
88
|
+
r"^scylla-enterprise$",
|
|
89
|
+
r"^enterprise-20\d{2}\.\d+$",
|
|
90
|
+
r"^siren-tests$",
|
|
91
|
+
]
|
|
92
|
+
|
|
71
93
|
def __init__(self) -> None:
|
|
72
94
|
super().__init__()
|
|
73
95
|
self._jenkins = jenkins.Jenkins(url=current_app.config["JENKINS_URL"],
|
|
74
96
|
username=current_app.config["JENKINS_USER"],
|
|
75
97
|
password=current_app.config["JENKINS_API_TOKEN"])
|
|
76
|
-
self._monitored_releases =
|
|
98
|
+
self._monitored_releases = self.JENKINS_MONITORED_RELEASES
|
|
99
|
+
|
|
100
|
+
def _check_release_name(self, release_name: str):
|
|
101
|
+
return any(re.match(pattern, release_name, re.IGNORECASE) for pattern in self._monitored_releases)
|
|
77
102
|
|
|
78
103
|
def collect(self):
|
|
79
104
|
click.echo("Collecting new tests from jenkins")
|
|
80
105
|
all_jobs = self._jenkins.get_all_jobs()
|
|
81
|
-
all_monitored_folders = [job for job in all_jobs if job["fullname"]
|
|
106
|
+
all_monitored_folders = [job for job in all_jobs if self._check_release_name(job["fullname"])]
|
|
107
|
+
LOGGER.info("Will collect %s", [f["fullname"] for f in all_monitored_folders])
|
|
108
|
+
|
|
82
109
|
for release in all_monitored_folders:
|
|
83
110
|
LOGGER.info("Processing release %s", release["name"])
|
|
84
111
|
try:
|
|
@@ -143,9 +170,12 @@ class JenkinsMonitor(ArgusTestsMonitor):
|
|
|
143
170
|
except StopIteration:
|
|
144
171
|
LOGGER.warning("Test %s for release %s (group %s) doesn't exist, creating...",
|
|
145
172
|
job["name"], saved_release.name, saved_group.name)
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
173
|
+
try:
|
|
174
|
+
saved_test = self.create_test(
|
|
175
|
+
saved_release, saved_group, job["name"], job["fullname"], job["url"])
|
|
176
|
+
self._existing_tests.append(saved_test)
|
|
177
|
+
except ArgusTestException:
|
|
178
|
+
LOGGER.error("Unable to create test for build_id %s", job["fullname"], exc_info=True)
|
|
149
179
|
|
|
150
180
|
def collect_groups_for_release(self, jobs):
|
|
151
181
|
# pylint: disable=no-self-use
|
|
@@ -1,12 +1,17 @@
|
|
|
1
|
+
import re
|
|
2
|
+
import requests
|
|
1
3
|
from typing import Any, TypedDict
|
|
4
|
+
from uuid import UUID
|
|
2
5
|
import xml.etree.ElementTree as ET
|
|
3
6
|
import jenkins
|
|
4
7
|
import logging
|
|
5
8
|
|
|
6
9
|
from flask import current_app, g
|
|
7
10
|
|
|
8
|
-
|
|
11
|
+
from argus.backend.models.web import ArgusGroup, ArgusRelease, ArgusTest, UserOauthToken
|
|
9
12
|
|
|
13
|
+
LOGGER = logging.getLogger(__name__)
|
|
14
|
+
GITHUB_REPO_RE = r"(?P<http>^https?:\/\/(www\.)?github\.com\/(?P<user>[\w\d\-]+)\/(?P<repo>[\w\d\-]+)(\.git)?$)|(?P<ssh>git@github\.com:(?P<ssh_user>[\w\d\-]+)\/(?P<ssh_repo>[\w\d\-]+)(\.git)?)"
|
|
10
15
|
|
|
11
16
|
class Parameter(TypedDict):
|
|
12
17
|
_class: str
|
|
@@ -15,9 +20,31 @@ class Parameter(TypedDict):
|
|
|
15
20
|
value: Any
|
|
16
21
|
|
|
17
22
|
|
|
23
|
+
class JenkinsServiceError(Exception):
|
|
24
|
+
pass
|
|
25
|
+
|
|
26
|
+
|
|
18
27
|
class JenkinsService:
|
|
19
28
|
RESERVED_PARAMETER_NAME = "requested_by_user"
|
|
20
29
|
|
|
30
|
+
SETTINGS_CONFIG_MAP = {
|
|
31
|
+
"scylla-cluster-tests": {
|
|
32
|
+
"gitRepo": "*//scm/userRemoteConfigs/hudson.plugins.git.UserRemoteConfig/url",
|
|
33
|
+
"gitBranch": "*//scm/branches/hudson.plugins.git.BranchSpec/name",
|
|
34
|
+
"pipelineFile": "*//scriptPath",
|
|
35
|
+
},
|
|
36
|
+
"driver-matrix-tests": {
|
|
37
|
+
"gitRepo": "*//scm/userRemoteConfigs/hudson.plugins.git.UserRemoteConfig/url",
|
|
38
|
+
"gitBranch": "*//scm/branches/hudson.plugins.git.BranchSpec/name",
|
|
39
|
+
"pipelineFile": "*//scriptPath",
|
|
40
|
+
},
|
|
41
|
+
"sirenada": {
|
|
42
|
+
"gitRepo": "*//scm/userRemoteConfigs/hudson.plugins.git.UserRemoteConfig/url",
|
|
43
|
+
"gitBranch": "*//scm/branches/hudson.plugins.git.BranchSpec/name",
|
|
44
|
+
"pipelineFile": "*//scriptPath",
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
21
48
|
def __init__(self) -> None:
|
|
22
49
|
self._jenkins = jenkins.Jenkins(url=current_app.config["JENKINS_URL"],
|
|
23
50
|
username=current_app.config["JENKINS_USER"],
|
|
@@ -42,6 +69,154 @@ class JenkinsService:
|
|
|
42
69
|
|
|
43
70
|
return params
|
|
44
71
|
|
|
72
|
+
def get_releases_for_clone(self, test_id: str):
|
|
73
|
+
test_id = UUID(test_id)
|
|
74
|
+
# TODO: Filtering based on origin location / user preferences
|
|
75
|
+
_: ArgusTest = ArgusTest.get(id=test_id)
|
|
76
|
+
|
|
77
|
+
releases = list(ArgusRelease.all())
|
|
78
|
+
|
|
79
|
+
return sorted(releases, key=lambda r: r.pretty_name if r.pretty_name else r.name)
|
|
80
|
+
|
|
81
|
+
def get_groups_for_release(self, release_id: str):
|
|
82
|
+
groups = list(ArgusGroup.filter(release_id=release_id).all())
|
|
83
|
+
|
|
84
|
+
return sorted(groups, key=lambda g: g.pretty_name if g.pretty_name else g.name)
|
|
85
|
+
|
|
86
|
+
def _verify_sct_settings(self, new_settings: dict[str, str]) -> tuple[bool, str]:
|
|
87
|
+
if not (match := re.match(GITHUB_REPO_RE, new_settings["gitRepo"])):
|
|
88
|
+
return (False, "Repository doesn't conform to GitHub schema")
|
|
89
|
+
|
|
90
|
+
git_info = match.groupdict()
|
|
91
|
+
if git_info.get("ssh"):
|
|
92
|
+
repo = git_info["ssh_repo"]
|
|
93
|
+
user = git_info["ssh_user"]
|
|
94
|
+
else:
|
|
95
|
+
repo = git_info["repo"]
|
|
96
|
+
user = git_info["user"]
|
|
97
|
+
|
|
98
|
+
user_tokens = UserOauthToken.filter(user_id=g.user.id).all()
|
|
99
|
+
token = None
|
|
100
|
+
for tok in user_tokens:
|
|
101
|
+
if tok.kind == "github":
|
|
102
|
+
token = tok.token
|
|
103
|
+
break
|
|
104
|
+
if not token:
|
|
105
|
+
raise JenkinsServiceError("Github token not found")
|
|
106
|
+
|
|
107
|
+
response = requests.get(
|
|
108
|
+
url=f"https://api.github.com/repos/{user}/{repo}/contents/{new_settings['pipelineFile']}?ref={new_settings['gitBranch']}",
|
|
109
|
+
headers={
|
|
110
|
+
"Accept": "application/vnd.github+json",
|
|
111
|
+
"Authorization": f"Bearer {token}",
|
|
112
|
+
}
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
if response.status_code == 404:
|
|
116
|
+
return (False, f"Pipeline file not found in the <a href=\"https://github.com/{user}/{repo}/tree/{new_settings['gitBranch']}\"> target repository</a>, please check the repository before continuing")
|
|
117
|
+
|
|
118
|
+
if response.status_code == 403:
|
|
119
|
+
return (True, "No access to this repository using your token. The pipeline file cannot be verified.")
|
|
120
|
+
|
|
121
|
+
if response.status_code == 200:
|
|
122
|
+
return (True, "")
|
|
123
|
+
|
|
124
|
+
return (False, "Generic Error")
|
|
125
|
+
|
|
126
|
+
def verify_job_settings(self, build_id: str, new_settings: dict[str, str]) -> tuple[bool, str]:
|
|
127
|
+
PLUGIN_MAP = {
|
|
128
|
+
"scylla-cluster-tests": self._verify_sct_settings,
|
|
129
|
+
# for now they match
|
|
130
|
+
"sirenada": self._verify_sct_settings,
|
|
131
|
+
"driver-matrix-tests": self._verify_sct_settings,
|
|
132
|
+
}
|
|
133
|
+
test: ArgusTest = ArgusTest.get(build_system_id=build_id)
|
|
134
|
+
plugin_name = test.plugin_name
|
|
135
|
+
|
|
136
|
+
validated, message = PLUGIN_MAP.get(plugin_name, lambda _: (True, ""))(new_settings)
|
|
137
|
+
|
|
138
|
+
return {
|
|
139
|
+
"validated": validated,
|
|
140
|
+
"message": message,
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
def get_advanced_settings(self, build_id: str):
|
|
144
|
+
test: ArgusTest = ArgusTest.get(build_system_id=build_id)
|
|
145
|
+
plugin_name = test.plugin_name
|
|
146
|
+
|
|
147
|
+
if not (plugin_settings := self.SETTINGS_CONFIG_MAP.get(plugin_name)):
|
|
148
|
+
return {}
|
|
149
|
+
|
|
150
|
+
settings = {}
|
|
151
|
+
raw_config = self._jenkins.get_job_config(name=build_id)
|
|
152
|
+
config = ET.fromstring(raw_config)
|
|
153
|
+
|
|
154
|
+
for setting, xpath in plugin_settings.items():
|
|
155
|
+
value = config.find(xpath)
|
|
156
|
+
settings[setting] = value.text
|
|
157
|
+
|
|
158
|
+
return settings
|
|
159
|
+
|
|
160
|
+
def adjust_job_settings(self, build_id: str, plugin_name: str, settings: dict[str, str]):
|
|
161
|
+
xpath_map = self.SETTINGS_CONFIG_MAP.get(plugin_name)
|
|
162
|
+
if not xpath_map:
|
|
163
|
+
return
|
|
164
|
+
|
|
165
|
+
config = self._jenkins.get_job_config(name=build_id)
|
|
166
|
+
xml = ET.fromstring(config)
|
|
167
|
+
for setting, value in settings.items():
|
|
168
|
+
element = xml.find(xpath_map[setting])
|
|
169
|
+
element.text = value
|
|
170
|
+
|
|
171
|
+
adjusted_config = ET.tostring(xml, encoding="unicode")
|
|
172
|
+
self._jenkins.reconfig_job(name=build_id, config_xml=adjusted_config)
|
|
173
|
+
|
|
174
|
+
def clone_job(self, current_test_id: str, new_name: str, target: str, group: str, advanced_settings: bool | dict[str, str]):
|
|
175
|
+
cloned_test: ArgusTest = ArgusTest.get(id=current_test_id)
|
|
176
|
+
target_release: ArgusRelease = ArgusRelease.get(id=target)
|
|
177
|
+
target_group: ArgusGroup = ArgusGroup.get(id=group)
|
|
178
|
+
|
|
179
|
+
if target_group.id == cloned_test.id and new_name == cloned_test.name:
|
|
180
|
+
raise JenkinsServiceError("Unable to clone: source and destination are the same")
|
|
181
|
+
|
|
182
|
+
if not target_group.build_system_id:
|
|
183
|
+
raise JenkinsServiceError("Unable to clone: target group is missing jenkins folder path")
|
|
184
|
+
|
|
185
|
+
jenkins_new_build_id = f"{target_group.build_system_id}/{new_name}"
|
|
186
|
+
|
|
187
|
+
new_test = ArgusTest()
|
|
188
|
+
new_test.name = new_name
|
|
189
|
+
new_test.build_system_id = jenkins_new_build_id
|
|
190
|
+
new_test.group_id = target_group.id
|
|
191
|
+
new_test.release_id = target_release.id
|
|
192
|
+
new_test.plugin_name = cloned_test.plugin_name
|
|
193
|
+
|
|
194
|
+
old_config = self._jenkins.get_job_config(name=cloned_test.build_system_id)
|
|
195
|
+
LOGGER.info(old_config)
|
|
196
|
+
xml = ET.fromstring(old_config)
|
|
197
|
+
display_name = xml.find("displayName")
|
|
198
|
+
if display_name:
|
|
199
|
+
display_name.text = new_name
|
|
200
|
+
new_config = ET.tostring(xml, encoding="unicode")
|
|
201
|
+
self._jenkins.create_job(name=jenkins_new_build_id, config_xml=new_config)
|
|
202
|
+
new_job_info = self._jenkins.get_job_info(name=jenkins_new_build_id)
|
|
203
|
+
new_test.build_system_url = new_job_info["url"]
|
|
204
|
+
new_test.save()
|
|
205
|
+
|
|
206
|
+
if advanced_settings:
|
|
207
|
+
self.adjust_job_settings(build_id=jenkins_new_build_id, plugin_name=new_test.plugin_name, settings=advanced_settings)
|
|
208
|
+
|
|
209
|
+
return {
|
|
210
|
+
"new_job": new_job_info,
|
|
211
|
+
"new_entity": new_test,
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
def clone_build_job(self, build_id: str, params: dict[str, str]):
|
|
215
|
+
queue_item = self.build_job(build_id=build_id, params=params)
|
|
216
|
+
return {
|
|
217
|
+
"queueItem": queue_item,
|
|
218
|
+
}
|
|
219
|
+
|
|
45
220
|
def build_job(self, build_id: str, params: dict, user_override: str = None):
|
|
46
221
|
queue_number = self._jenkins.build_job(build_id, {
|
|
47
222
|
**params,
|
|
@@ -44,6 +44,20 @@ class ReleaseManagerService:
|
|
|
44
44
|
def get_tests(self, group_id: UUID) -> list[ArgusTest]:
|
|
45
45
|
return list(ArgusTest.filter(group_id=group_id).all())
|
|
46
46
|
|
|
47
|
+
def toggle_test_enabled(self, test_id: UUID, new_state: bool) -> bool:
|
|
48
|
+
test: ArgusTest = ArgusTest.get(id=test_id)
|
|
49
|
+
test.enabled = new_state
|
|
50
|
+
test.save()
|
|
51
|
+
|
|
52
|
+
return test
|
|
53
|
+
|
|
54
|
+
def toggle_group_enabled(self, group_id: UUID, new_state: bool) -> bool:
|
|
55
|
+
test: ArgusGroup = ArgusGroup.get(id=group_id)
|
|
56
|
+
test.enabled = new_state
|
|
57
|
+
test.save()
|
|
58
|
+
|
|
59
|
+
return test
|
|
60
|
+
|
|
47
61
|
def create_release(self, release_name: str, pretty_name: str, perpetual: bool) -> ArgusRelease:
|
|
48
62
|
try:
|
|
49
63
|
release = ArgusRelease.get(name=release_name)
|