argus-alm 0.11.7__tar.gz → 0.12.1__tar.gz

{argus_alm-0.11.7 → argus_alm-0.12.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: argus-alm
-Version: 0.11.7
+Version: 0.12.1
 Summary: Argus
 Home-page: https://github.com/scylladb/argus
 License: Apache-2.0
@@ -11,6 +11,7 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Requires-Dist: click (>=8.1.3,<9.0.0)
 Requires-Dist: requests (>=2.26.0,<3.0.0)
 Project-URL: Repository, https://github.com/scylladb/argus
 Description-Content-Type: text/markdown

{argus_alm-0.11.7 → argus_alm-0.12.1}/argus/backend/controller/admin_api.py

@@ -6,7 +6,7 @@ from flask import (
     Request,
 )
 from argus.backend.error_handlers import handle_api_exception
-from argus.backend.service.release_manager import ReleaseManagerService
+from argus.backend.service.release_manager import ReleaseEditPayload, ReleaseManagerService
 from argus.backend.service.user import api_login_required, check_roles
 from argus.backend.models.web import UserRoles
 
@@ -92,6 +92,36 @@ def set_release_dormancy():
     }
 
 
+@bp.route("/release/edit", methods=["POST"])
+@check_roles(UserRoles.Admin)
+@api_login_required
+def edit_release():
+    payload: ReleaseEditPayload = get_payload(request)
+    result = ReleaseManagerService().edit_release(payload)
+
+    return {
+        "status": "ok",
+        "response": {
+            "updated": result
+        }
+    }
+
+
+@bp.route("/release/delete", methods=["POST"])
+@check_roles(UserRoles.Admin)
+@api_login_required
+def delete_release():
+    payload = get_payload(request)
+    result = ReleaseManagerService().delete_release(release_id=payload["releaseId"])
+
+    return {
+        "status": "ok",
+        "response": {
+            "deleted": result
+        }
+    }
+
+
 @bp.route("/group/create", methods=["POST"])
 @check_roles(UserRoles.Admin)
 @api_login_required

{argus_alm-0.11.7 → argus_alm-0.12.1}/argus/backend/controller/api.py

@@ -389,9 +389,10 @@ def release_stats_v2():
     release = request.args.get("release")
     limited = bool(int(request.args.get("limited")))
     version = request.args.get("productVersion", None)
+    include_no_version = bool(int(request.args.get("includeNoVersion", True)))
     force = bool(int(request.args.get("force")))
     stats = ReleaseStatsCollector(
-        release_name=release, release_version=version).collect(limited=limited, force=force)
+        release_name=release, release_version=version).collect(limited=limited, force=force, include_no_version=include_no_version)
 
     res = jsonify({
         "status": "ok",

{argus_alm-0.11.7 → argus_alm-0.12.1}/argus/backend/controller/client_api.py

@@ -81,7 +81,11 @@ def run_submit_logs(run_type: str, run_id: str):
 @bp.route("/testrun/<string:run_type>/<string:run_id>/finalize", methods=["POST"])
 @api_login_required
 def run_finalize(run_type: str, run_id: str):
-    result = ClientService().finish_run(run_type=run_type, run_id=run_id)
+    try:
+        payload = get_payload(request)
+    except Exception:
+        payload = None
+    result = ClientService().finish_run(run_type=run_type, run_id=run_id, payload=payload)
     return {
         "status": "ok",
         "response": result

{argus_alm-0.11.7 → argus_alm-0.12.1}/argus/backend/models/web.py

@@ -111,6 +111,7 @@ class ArgusRelease(Model):
     pretty_name = columns.Text()
     description = columns.Text()
     github_repo_url = columns.Text()
+    valid_version_regex = columns.Text()
     assignee = columns.List(value_type=columns.UUID)
     picture_id = columns.UUID()
     enabled = columns.Boolean(default=lambda: True)

{argus_alm-0.11.7 → argus_alm-0.12.1}/argus/backend/plugins/core.py

@@ -38,6 +38,7 @@ class PluginModelBase(Model):
     heartbeat = columns.Integer(default=lambda: int(time()))
     end_time = columns.DateTime(default=lambda: datetime.utcfromtimestamp(0))
     build_job_url = columns.Text()
+    product_version = columns.Text(index=True)
 
     # Test Logs Collection
     logs = columns.List(value_type=columns.Tuple(columns.Text(), columns.Text()))
@@ -180,10 +181,13 @@ class PluginModelBase(Model):
     def submit_product_version(self, version: str):
         raise NotImplementedError()
 
+    def set_product_version(self, version: str):
+        self.product_version = version
+
     def submit_logs(self, logs: list[dict]):
         raise NotImplementedError()
 
-    def finish_run(self):
+    def finish_run(self, payload: dict = None):
         raise NotImplementedError()
 
 

argus_alm-0.12.1/argus/backend/plugins/driver_matrix_tests/controller.py

@@ -0,0 +1,24 @@
+from flask import Blueprint, request
+
+from argus.backend.error_handlers import handle_api_exception
+from argus.backend.service.user import api_login_required
+from argus.backend.plugins.driver_matrix_tests.service import DriverMatrixService
+
+bp = Blueprint("driver_matrix_api", __name__, url_prefix="/driver_matrix")
+bp.register_error_handler(Exception, handle_api_exception)
+
+
+@bp.route("/test_report", methods=["GET"])
+@api_login_required
+def driver_matrix_test_report():
+
+    build_id = request.args.get("buildId")
+    if not build_id:
+        raise Exception("No build id provided")
+    
+
+    result = DriverMatrixService().tested_versions_report(build_id=build_id)
+    return {
+        "status": "ok",
+        "response": result
+    }

{argus_alm-0.11.7 → argus_alm-0.12.1}/argus/backend/plugins/driver_matrix_tests/model.py

@@ -166,7 +166,7 @@ class DriverTestRun(PluginModelBase):
     def submit_product_version(self, version: str):
         self.scylla_version = version
 
-    def finish_run(self):
+    def finish_run(self, payload: dict = None):
         self.end_time = datetime.utcnow()
 
     def submit_logs(self, logs: list[dict]):

{argus_alm-0.11.7 → argus_alm-0.12.1}/argus/backend/plugins/driver_matrix_tests/plugin.py

@@ -2,6 +2,7 @@ from flask import Blueprint
 
 from argus.backend.plugins.core import PluginInfoBase, PluginModelBase
 from argus.backend.plugins.driver_matrix_tests.model import DriverTestRun
+from argus.backend.plugins.driver_matrix_tests.controller import bp as driver_matrix_api_bp
 from argus.backend.plugins.driver_matrix_tests.udt import TestCollection, EnvironmentInfo, TestCase, TestSuite
 
 
@@ -9,7 +10,7 @@ class PluginInfo(PluginInfoBase):
     # pylint: disable=too-few-public-methods
     name: str = "driver-matrix-tests"
     model: PluginModelBase = DriverTestRun
-    controller: Blueprint = None
+    controller: Blueprint = driver_matrix_api_bp
     all_models = [
         DriverTestRun,
     ]

argus_alm-0.12.1/argus/backend/plugins/driver_matrix_tests/service.py

@@ -0,0 +1,42 @@
+from argus.backend.db import ScyllaCluster
+from argus.backend.models.web import ArgusRelease, ArgusTest
+from argus.backend.plugins.driver_matrix_tests.model import DriverTestRun
+
+
+class DriverMatrixService:
+    def tested_versions_report(self, build_id: str) -> dict:
+        db = ScyllaCluster.get()
+        all_runs_for_test_query = db.prepare(f"SELECT * FROM {DriverTestRun.table_name()} WHERE build_id = ?")
+
+        rows = list(db.session.execute(all_runs_for_test_query, parameters=(build_id,)).all())
+
+        if len(rows) == 0:
+            raise Exception(f"No results for build_id {build_id}", build_id)
+
+        latest = rows[0]
+        try:
+            test: ArgusTest = ArgusTest.get(id=latest["test_id"])
+            release: ArgusRelease = ArgusRelease.get(id=latest["release_id"])
+        except (ArgusTest.DoesNotExist, ArgusRelease.DoesNotExist):
+            raise Exception(f"Unable to find release and test information for build_id {build_id} and run_id {latest['id']}", build_id, latest["id"])
+        
+
+        version_map = {}
+
+        for row in rows:
+            driver_versions = [(col["name"], col["driver"]) for col in row["test_collection"]]
+            for version, driver_type in driver_versions:
+                driver_type = driver_type if driver_type else "Unknown"
+                versions: list = version_map.get(driver_type, [])
+                if version not in versions:
+                    versions.append(version)
+                    versions.sort()
+                version_map[driver_type] = versions
+
+        response = {
+            "release": release.name,
+            "test": test.name,
+            "build_id": build_id,
+            "versions": version_map,
+        }
+        return response

argus_alm-0.12.1/argus/backend/plugins/generic/model.py

@@ -0,0 +1,76 @@
+from datetime import datetime
+from uuid import UUID
+from cassandra.cqlengine import columns
+from cassandra.cqlengine.models import Model
+from argus.backend.db import ScyllaCluster
+from argus.backend.models.web import ArgusRelease
+from argus.backend.plugins.core import PluginModelBase
+from argus.backend.plugins.generic.types import GenericRunFinishRequest, GenericRunSubmitRequest
+from argus.backend.util.enums import TestStatus
+
+
+class GenericPluginException(Exception):
+    pass
+
+
+class GenericRun(PluginModelBase):
+    _plugin_name = "generic"
+    __table_name__ = "generic_run"
+    logs = columns.Map(key_type=columns.Text(), value_type=columns.Text())
+    started_by = columns.Text()
+    # TODO: Legacy field name, should be renamed to product_version and abstracted
+    scylla_version = columns.Text()
+
+    @classmethod
+    def _stats_query(cls) -> str:
+        return ("SELECT id, test_id, group_id, release_id, status, start_time, build_job_url, build_id, "
+                f"assignee, end_time, investigation_status, heartbeat, scylla_version FROM {cls.table_name()} WHERE release_id = ?")
+
+    @classmethod
+    def get_distinct_product_versions(cls, release: ArgusRelease, cluster: ScyllaCluster = None) -> list[str]:
+        if not cluster:
+            cluster = ScyllaCluster.get()
+        statement = cluster.prepare(f"SELECT scylla_version FROM {cls.table_name()} WHERE release_id = ?")
+        rows = cluster.session.execute(query=statement, parameters=(release.id,))
+        unique_versions = {r["scylla_version"] for r in rows if r["scylla_version"]}
+
+        return sorted(list(unique_versions), reverse=True)
+
+    def submit_product_version(self, version: str):
+        self.scylla_version = version
+        self.set_product_version(version)
+
+    @classmethod
+    def load_test_run(cls, run_id: UUID) -> 'GenericRun':
+        return cls.get(id=run_id)
+
+    @classmethod
+    def submit_run(cls, request_data: GenericRunSubmitRequest) -> 'GenericRun':
+        try:
+            run = cls.get(id=request_data["run_id"])
+            raise GenericPluginException(f"Run with UUID {request_data['run_id']} already exists.", request_data["run_id"])
+        except cls.DoesNotExist:
+            pass
+        run = cls()
+        run.start_time = datetime.utcnow()
+        run.build_id = request_data["build_id"]
+        run.started_by = request_data["started_by"]
+        run.id = request_data["run_id"]
+        run.build_job_url = request_data["build_url"]
+        run.assign_categories()
+        try:
+            run.assignee = run.get_scheduled_assignee()
+        except Model.DoesNotExist:
+            run.assignee = None
+        if version := request_data.get("scylla_version"):
+            run.submit_product_version(version)
+        run.status = TestStatus.RUNNING.value
+        run.save()
+
+        return run
+
+    def finish_run(self, payload: GenericRunFinishRequest = None):
+        self.end_time = datetime.utcnow()
+        self.status = TestStatus(payload["status"]).value
+        if version := payload.get("scylla_version"):
+            self.submit_product_version(version)

argus_alm-0.12.1/argus/backend/plugins/generic/plugin.py

@@ -0,0 +1,16 @@
+from flask import Blueprint
+
+from argus.backend.plugins.core import PluginInfoBase, PluginModelBase
+from argus.backend.plugins.generic.model import GenericRun
+
+
+class PluginInfo(PluginInfoBase):
+    # pylint: disable=too-few-public-methods
+    name: str = "generic"
+    model: PluginModelBase = GenericRun
+    controller: Blueprint = None
+    all_models = [
+        GenericRun
+    ]
+    all_types = [
+    ]

argus_alm-0.12.1/argus/backend/plugins/generic/types.py

@@ -0,0 +1,13 @@
+from typing import TypedDict
+
+class GenericRunSubmitRequest(TypedDict):
+    build_id: str
+    build_url: str
+    run_id: str
+    started_by: str
+    scylla_version: str | None
+
+
+class GenericRunFinishRequest(TypedDict):
+    status: str
+    scylla_version: str | None

{argus_alm-0.11.7 → argus_alm-0.12.1}/argus/backend/plugins/sct/service.py

@@ -383,7 +383,7 @@ class SCTService:
                     "aborted": 0,
                 }
             )
-            if run["status"] in ["passed", "failed", "aborted"]:
+            if run["status"] in ["passed", "failed", "aborted", "test_error"]:
                 metadata[run["status"]] += 1
             kernel_metadata[kernel_package["version"]] = metadata
 

{argus_alm-0.11.7 → argus_alm-0.12.1}/argus/backend/plugins/sct/testrun.py

@@ -47,9 +47,9 @@ class SCTTestRunSubmissionRequest():
     job_url: str
     started_by: str
     commit_id: str
-    origin_url: str | None
-    branch_name: str | None
     sct_config: dict | None
+    origin_url: Optional[str] = field(default=None)
+    branch_name: Optional[str] = field(default=None)
     runner_public_ip: Optional[str] = field(default=None)
     runner_private_ip: Optional[str] = field(default=None)
 
@@ -60,6 +60,7 @@ class SCTTestRun(PluginModelBase):
 
     # Test Details
     test_name = columns.Text()
+    stress_duration = columns.Float()
     scm_revision_id = columns.Text()
     branch_name = columns.Text()
     origin_url = columns.Text()
@@ -181,6 +182,8 @@ class SCTTestRun(PluginModelBase):
 
         if req.sct_config:
             backend = req.sct_config.get("cluster_backend")
+            if duration_override := req.sct_config.get("stress_duration"):
+                run.stress_duration = float(duration_override)
             region_key = SCT_REGION_PROPERTY_MAP.get(backend, SCT_REGION_PROPERTY_MAP["default"])
             raw_regions = req.sct_config.get(region_key) or "undefined_region"
             regions = raw_regions.split() if isinstance(raw_regions, str) else raw_regions
@@ -212,7 +215,7 @@ class SCTTestRun(PluginModelBase):
     def submit_product_version(self, version: str):
         self.scylla_version = version
 
-    def finish_run(self):
+    def finish_run(self, payload: dict = None):
         self.end_time = datetime.utcnow()
 
     def submit_logs(self, logs: list[dict]):

{argus_alm-0.11.7 → argus_alm-0.12.1}/argus/backend/plugins/sct/udt.py

@@ -20,6 +20,8 @@ class CloudInstanceDetails(UserType):
     region = columns.Text()
     public_ip = columns.Text()
     private_ip = columns.Text()
+    dc_name = columns.Text()
+    rack_name = columns.Text()
     creation_time = columns.Integer(default=lambda: int(time()))
     termination_time = columns.Integer(default=lambda: 0)
     termination_reason = columns.Text(default=lambda: "")

{argus_alm-0.11.7 → argus_alm-0.12.1}/argus/backend/plugins/sirenada/model.py

@@ -65,7 +65,7 @@ class SirenadaRun(PluginModelBase):
     def submit_logs(self, logs: dict[str, str]):
         raise SirenadaPluginException("Log submission is not supported for Sirenada")
 
-    def finish_run(self):
+    def finish_run(self, payload: dict = None):
         raise SirenadaPluginException("Sirenada runs do not need finalization")
 
     @classmethod

{argus_alm-0.11.7 → argus_alm-0.12.1}/argus/backend/service/client_service.py

@@ -62,10 +62,10 @@ class ClientService:
 
         return "Submitted"
 
-    def finish_run(self, run_type: str, run_id: str) -> str:
+    def finish_run(self, run_type: str, run_id: str, payload: dict | None = None) -> str:
         model = self.get_model(run_type)
         run = model.load_test_run(UUID(run_id))
-        run.finish_run()
+        run.finish_run(payload)
         run.save()
 
         return "Finalized"

{argus_alm-0.11.7 → argus_alm-0.12.1}/argus/backend/service/release_manager.py

@@ -1,4 +1,5 @@
 import logging
+from typing import TypedDict
 from uuid import UUID
 from argus.backend.db import ScyllaCluster
 from argus.backend.plugins.sct.testrun import SCTTestRun
@@ -11,6 +12,16 @@ class ReleaseManagerException(Exception):
     pass
 
 
+class ReleaseEditPayload(TypedDict):
+    id: str
+    pretty_name: str
+    description: str
+    valid_version_regex: str | None
+    enabled: bool
+    perpetual: bool
+    dormant: bool
+
+
 class ReleaseManagerService:
     def __init__(self) -> None:
         self.session = ScyllaCluster.get_session()
@@ -156,6 +167,33 @@ class ReleaseManagerService:
 
         return True
 
+    def edit_release(self, payload: ReleaseEditPayload) -> bool:
+
+        release: ArgusRelease = ArgusRelease.get(id=payload["id"])
+        release.pretty_name = payload["pretty_name"]
+        release.perpetual = payload["perpetual"]
+        release.enabled = payload["enabled"]
+        release.dormant = payload["dormant"]
+        release.description = payload["description"]
+        release.valid_version_regex = payload["valid_version_regex"]
+
+
+        release.save()
+        return True
+
+    def delete_release(self, release_id: str) -> bool:
+
+        release: ArgusRelease = ArgusRelease.get(id=release_id)
+
+        release_groups = ArgusGroup.filter(release_id=release.id)
+        release_tests = ArgusTest.filter(release_id=release.id)
+
+        for entity in [*release_groups.all(), *release_tests.all()]:
+            entity.delete()
+
+        release.delete()
+        return True
+
     def batch_move_tests(self, new_group_id: str, tests: list[str]) -> bool:
         group = ArgusGroup.get(id=UUID(new_group_id))
 

{argus_alm-0.11.7 → argus_alm-0.12.1}/argus/backend/service/stats.py

@@ -33,6 +33,7 @@ class TestRunStatRow(TypedDict):
 class ComparableTestStatus:
     PRIORITY_MAP = {
         TestStatus.FAILED: 10,
+        TestStatus.TEST_ERROR: 10,
         TestStatus.ABORTED: 9,
         TestStatus.RUNNING: 8,
         TestStatus.CREATED: 7,
@@ -126,7 +127,7 @@ def generate_field_status_map(
         field_name = "status",
         container_class = TestStatus,
         cmp_class = ComparableTestStatus
-    ) -> dict[int, str]:
+    ) -> dict[int, tuple[str, TestRunStatRow]]:
 
     status_map = {}
     for run in last_runs:
@@ -136,7 +137,7 @@ def generate_field_status_map(
                 if cmp_class(container_class(status)) < cmp_class(container_class(run[field_name])):
                     status_map[run_number] = run[field_name]
             case _:
-                status_map[run_number] = run[field_name]
+                status_map[run_number] = (run[field_name], run)
     return status_map
 
 
@@ -279,6 +280,7 @@ class TestStats:
         self.has_comments = False
         self.schedules = schedules if schedules else tuple()
         self.is_scheduled = len(self.schedules) > 0
+        self.tracked_run_number = None
 
     def to_dict(self) -> dict:
         return {
@@ -288,7 +290,8 @@ class TestStats:
             "last_runs": self.last_runs,
             "start_time": self.start_time,
             "hasBugReport": self.has_bug_report,
-            "hasComments": self.has_comments
+            "hasComments": self.has_comments,
+            "buildNumber": self.tracked_run_number,
         }
 
     def collect(self, limited=False):
@@ -305,11 +308,10 @@ class TestStats:
             self.parent_group.increment_status(status=self.status)
             return
         status_map = generate_field_status_map(last_runs)
-        investigation_status_map = generate_field_status_map(
-            last_runs, "investigation_status", TestInvestigationStatus, ComparableTestInvestigationStatus)
 
-        self.status = status_map.get(get_build_number(last_run["build_job_url"]))
-        self.investigation_status = investigation_status_map.get(get_build_number(last_run["build_job_url"]))
+        worst_case = status_map.get(get_build_number(last_run["build_job_url"]))
+        self.status = worst_case[0]
+        self.investigation_status = worst_case[1]["investigation_status"]
         self.start_time = last_run["start_time"]
 
         self.parent_group.increment_status(status=self.status)
@@ -318,6 +320,7 @@ class TestStats:
 
         self.last_runs = [
             {
+                "id": run["id"],
                 "status": run["status"],
                 "build_number": get_build_number(run["build_job_url"]),
                 "build_job_name": run["build_id"],
@@ -327,10 +330,18 @@ class TestStats:
                 "comments": [dict(i.items()) for i in self.parent_group.parent_release.comments if i.test_run_id == run["id"]],
             }
             for run in last_runs
-        ][:5]
-        self.has_bug_report = len(self.last_runs[0]["issues"]) > 0
+        ]
+        try:
+            target_run = next(run for run in self.last_runs if run["id"] == worst_case[1]["id"])
+        except StopIteration:
+            target_run = worst_case[1]
+            target_run["issues"] = [dict(i.items()) for i in self.parent_group.parent_release.issues if i.run_id == target_run["id"]]
+            target_run["comments"] = [dict(i.items()) for i in self.parent_group.parent_release.comments if i.test_run_id == target_run["id"]]
+        self.has_bug_report = len(target_run["issues"]) > 0
         self.parent_group.parent_release.has_bug_report = self.has_bug_report or self.parent_group.parent_release.has_bug_report
-        self.has_comments = len(self.last_runs[0]["comments"]) > 0
+        self.has_comments = len(target_run["comments"]) > 0
+        self.last_runs = self.last_runs[:5]
+        self.tracked_run_number = target_run.get("build_number", get_build_number(target_run.get("build_job_url")))
 
 
 class ReleaseStatsCollector:
@@ -343,7 +354,7 @@ class ReleaseStatsCollector:
         self.release_name = release_name
         self.release_version = release_version
 
-    def collect(self, limited=False, force=False) -> dict:
+    def collect(self, limited=False, force=False, include_no_version=False) -> dict:
         self.release: ArgusRelease = ArgusRelease.get(name=self.release_name)
         self.release_rows = [row for plugin in all_plugin_models()
                              for row in plugin.get_stats_for_release(release=self.release)]
@@ -353,8 +364,18 @@ class ReleaseStatsCollector:
             }
 
         if self.release_version:
-            self.release_rows = list(
-                filter(lambda row: row["scylla_version"] == self.release_version or not row["scylla_version"], self.release_rows))
+            if include_no_version:
+                expr = lambda row: row["scylla_version"] == self.release_version or not row["scylla_version"] 
+            elif self.release_version == "!noVersion":
+                expr = lambda row: not row["scylla_version"]
+            else:
+                expr = lambda row: row["scylla_version"] == self.release_version
+        else:
+            if include_no_version:
+                expr = lambda row: row
+            else:
+                expr = lambda row: row["scylla_version"]
+        self.release_rows = list(filter(expr, self.release_rows))
 
         self.release_stats = ReleaseStats(release=self.release)
         self.release_stats.collect(rows=self.release_rows, limited=limited, force=force)

{argus_alm-0.11.7 → argus_alm-0.12.1}/argus/backend/service/testrun.py

@@ -341,7 +341,7 @@ class TestRunService:
             }
         )
         if issue_request.status_code != 200:
-            raise Exception(f"Error getting issue state: Response: HTTP {issue_request.status_code}", issue_request)
+            raise Exception(f"Error getting issue state: Response: HTTP {issue_request.status_code}", issue_request.json())
 
         issue_state: dict[str, Any] = issue_request.json()
 

{argus_alm-0.11.7 → argus_alm-0.12.1}/argus/backend/util/enums.py

@@ -14,6 +14,7 @@ class TestStatus(str, Enum):
     CREATED = "created"
     RUNNING = "running"
     FAILED = "failed"
+    TEST_ERROR = "test_error"
     ERROR = "error"
     PASSED = "passed"
     ABORTED = "aborted"

{argus_alm-0.11.7 → argus_alm-0.12.1}/argus/client/base.py

@@ -170,12 +170,14 @@ class ArgusClient:
             }
         )
 
-    def finalize_run(self, run_type: str, run_id: UUID) -> requests.Response:
+    def finalize_run(self, run_type: str, run_id: UUID, body: dict = None) -> requests.Response:
+        body = body if body else {}
         return self.post(
             endpoint=self.Routes.FINALIZE,
             location_params={"type": run_type, "id": str(run_id)},
             body={
                 **self.generic_body,
+                **body,
             }
         )
 

argus_alm-0.12.1/argus/client/generic/cli.py

@@ -0,0 +1,47 @@
+import click
+import logging
+from argus.backend.util.enums import TestStatus
+
+from argus.client.generic.client import ArgusGenericClient
+
+LOGGER = logging.getLogger(__name__)
+
+
+@click.group
+def cli():
+    pass
+
+
+@click.command("submit")
+@click.option("--api-key", help="Argus API key for authorization", required=True)
+@click.option("--base-url", default="https://argus.scylladb.com", help="Base URL for argus instance")
+@click.option("--id", required=True, help="UUID (v4 or v1) unique to the job")
+@click.option("--build-id", required=True, help="Unique job identifier in the build system, e.g. scylla-master/group/job for jenkins (The full path)")
+@click.option("--build-url", required=True, help="Job URL in the build system")
+@click.option("--started-by", required=True, help="Username of the user who started the job")
+@click.option("--scylla-version", required=False, default=None, help="Version of Scylla used for this job")
+def submit_run(api_key: str, base_url: str, id: str, build_id: str, build_url: str, started_by: str, scylla_version: str = None):
+    LOGGER.info("Submitting %s (%s) to Argus...", build_id, id)
+    client = ArgusGenericClient(auth_token=api_key, base_url=base_url)
+    client.submit_generic_run(build_id=build_id, run_id=id, started_by=started_by, build_url=build_url, scylla_version=scylla_version)
+    LOGGER.info("Done.")
+
+
+@click.command("finish")
+@click.option("--api-key", help="Argus API key for authorization", required=True)
+@click.option("--base-url", default="https://argus.scylladb.com", help="Base URL for argus instance")
+@click.option("--id", required=True, help="UUID (v4 or v1) unique to the job")
+@click.option("--status", required=True, help="Resulting job status")
+@click.option("--scylla-version", required=False, default=None, help="Version of Scylla used for this job")
+def finish_run(api_key: str, base_url: str, id: str, status: str, scylla_version: str = None):
+    client = ArgusGenericClient(auth_token=api_key, base_url=base_url)
+    status = TestStatus(status)
+    client.finalize_generic_run(run_id=id, status=status, scylla_version=scylla_version)
+
+
+cli.add_command(submit_run)
+cli.add_command(finish_run)
+
+
+if __name__ == "__main__":
+    cli()

argus_alm-0.12.1/argus/client/generic/client.py

@@ -0,0 +1,31 @@
+import logging
+from argus.client.base import ArgusClient
+
+
+LOGGER = logging.getLogger(__name__)
+
+class ArgusGenericClient(ArgusClient):
+    test_type = "generic"
+    schema_version: None = "v1"
+    def __init__(self, auth_token: str, base_url: str, api_version="v1") -> None:
+        super().__init__(auth_token, base_url, api_version)
+
+    def submit_generic_run(self, build_id: str, run_id: str, started_by: str, build_url: str, scylla_version: str | None = None):
+        request_body = {
+            "build_id": build_id,
+            "run_id": run_id,
+            "started_by": started_by,
+            "build_url": build_url,
+            "scylla_version": scylla_version
+        }
+
+        response = self.submit_run(run_type=self.test_type, run_body=request_body)
+        self.check_response(response)
+
+
+    def finalize_generic_run(self, run_id: str, status: str, scylla_version: str | None = None):
+        response = self.finalize_run(run_type=self.test_type, run_id=run_id, body={
+            "status": status,
+            "scylla_version": scylla_version,
+        })
+        self.check_response(response)

{argus_alm-0.11.7 → argus_alm-0.12.1}/argus/client/sct/client.py

@@ -148,7 +148,7 @@ class ArgusSCTClient(ArgusClient):
         self.check_response(response)
 
     def create_resource(self, name: str, resource_type: str, public_ip: str, private_ip: str,
-                        region: str, provider: str, shards_amount: int, state=ResourceState.RUNNING) -> None:
+                        region: str, provider: str, dc_name: str, rack_name: str, shards_amount: int, state=ResourceState.RUNNING) -> None:
         """
             Creates a cloud resource record in argus.
         """
@@ -164,6 +164,8 @@ class ArgusSCTClient(ArgusClient):
                     "instance_details": {
                         "provider": provider,
                         "region": region,
+                        "dc_name": dc_name,
+                        "rack_name": rack_name,
                         "public_ip": public_ip,
                         "private_ip": private_ip,
                         "shards_amount": shards_amount,

{argus_alm-0.11.7 → argus_alm-0.12.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "argus-alm"
-version = "0.11.7"
+version = "0.12.1"
 description = "Argus"
 authors = ["Alexey Kartashov <alexey.kartashov@scylladb.com>"]
 license = "Apache-2.0"
@@ -17,6 +17,10 @@ optional = true
 [tool.poetry.dependencies]
 requests = "^2.26.0"
 python = "^3.10"
+click = "^8.1.3"
+
+[tool.poetry.scripts]
+argus-client-generic = 'argus.client.generic.cli:cli'
 
 [tool.poetry.group.web-backend.dependencies]
 PyYAML = "^5.4.1"

argus_alm-0.11.7/setup.py

@@ -1,45 +0,0 @@
-# -*- coding: utf-8 -*-
-from setuptools import setup
-
-packages = \
-['argus',
- 'argus.backend',
- 'argus.backend.controller',
- 'argus.backend.events',
- 'argus.backend.models',
- 'argus.backend.plugins',
- 'argus.backend.plugins.driver_matrix_tests',
- 'argus.backend.plugins.sct',
- 'argus.backend.plugins.sirenada',
- 'argus.backend.service',
- 'argus.backend.util',
- 'argus.client',
- 'argus.client.driver_matrix_tests',
- 'argus.client.sct',
- 'argus.client.sirenada',
- 'argus.db']
-
-package_data = \
-{'': ['*']}
-
-install_requires = \
-['requests>=2.26.0,<3.0.0']
-
-setup_kwargs = {
-    'name': 'argus-alm',
-    'version': '0.11.7',
-    'description': 'Argus',
-    'long_description': '# Argus\n\n## Description\n\nArgus is a test tracking system intended to provide observability into automated test pipelines which use long-running resources. It allows observation of a test status, its events and its allocated resources. It also allows easy comparison between particular runs of a specific test.\n\n## Installation notes\n\n### Prerequisites\n\n- Python >=3.10.0 (system-wide or pyenv)\n\n- NodeJS >=16 (with npm)\n\n- Yarn (can be installed globally with `npm -g install yarn`)\n\n- nginx\n\n- poetry >=1.2.0b1\n\n### From source\n\n#### Production\n\nPerform the following steps:\n\nCreate a user that will be used by uwsgi:\n\n```bash\nuseradd -m -s /bin/bash argus\nsudo -iu argus\n```\n\n(Optional) Install pyenv and create a virtualenv for this user:\n\n```bash\npyenv install 3.10.0\npyenv virtualenv argus\npyenv activate argus\n```\n\nClone the project into a directory somewhere where user has full write permissions\n\n```bash\ngit clone https://github.com/scylladb/argus ~/app\ncd ~/app\n```\n\nInstall project dependencies:\n\n```bash\npoetry install --with default,dev,web-backend,docker-image\nyarn install\n```\n\nCompile frontend files from `/frontend` into `/public/dist`\n\n```bash\nyarn webpack\n```\n\nCreate a `argus.local.yaml` configuration file (used to configure database connection) and a `argus_web.yaml` (used for webapp secrets) in your application install directory.\n\n```bash\ncp argus_web.example.yaml argus_web.yaml\ncp argus.yaml argus.local.yaml\n```\n\nOpen `argus.local.yaml` and add the database connection information (contact_points, user, password and keyspace name).\n\nOpen `argus_web.yaml` and change the `SECRET_KEY` value to something secure, like a sha512 digest of random bytes. Fill out GITHUB_* variables with their respective values.\n\nCopy nginx configuration file from `docs/configs/argus.nginx.conf` to nginx virtual hosts directory:\n\nUbuntu:\n\n```bash\nsudo cp docs/configs/argus.nginx.conf /etc/nginx/sites-available/argus\nsudo ln -s /etc/nginx/sites-enabled/argus /etc/nginx/sites-available/argus\n```\n\nRHEL/Centos/Alma/Fedora:\n\n```bash\nsudo cp docs/configs/argus.nginx.conf /etc/nginx/conf.d/argus.conf\n```\n\nAdjust the webhost settings in that file as necessary, particularly `listen` and `server_name` directives.\n\nCopy systemd service file from `docs/config/argus.service` to `/etc/systemd/system` directory:\n\n```bash\nsudo cp docs/config/argus.service /etc/systemd/system\n```\n\nOpen it and adjust the path to the `start_argus.sh` script in the `ExecStart=` directive and the user/group, then reload systemd daemon configuration and enable (and optionally start) the service.\n\nWARNING: `start_argus.sh` assumes pyenv is installed into `~/.pyenv`\n\n```bash\nsudo systemctl daemon-reload\nsudo systemctl enable --now argus.service\n```\n\n#### Development\n\nClone the project into a directory somewhere\n\n```bash\ngit clone https://github.com/scylladb/argus\ncd argus\n```\n\nInstall project dependencies:\n\n```bash\npoetry install --with default,dev,web-backend,docker-image\nyarn install\n```\n\nCompile frontend files from `/frontend` into `/public/dist`. Add --watch to recompile files on change.\n\n```bash\nyarn webpack --watch\n```\n##### Configuration\nCreate a `argus.local.yaml` configuration file (used to configure database connection) and a `argus_web.yaml` (used for webapp secrets) in your application install directory.\n\nSee `Production` section for more details.\nTo configure Github authentication follow steps:\n1. Authorize OAuth App\n   1. go to your Account Settings (top right corner) -> Developer settings (left pane) -> OAuth Apps\n   2. Click Create New OAuth App button\n   3. Fill the fields (app name: `argus-dev`, homepage URL `http://localhost:5000`, Auth callback URL: `http://localhost:5000/profile/oauth/github`)\n   4. Confirm and get the tokens/ids required for config\n2. Create Jenkins token for your account\n   1. Go to `Configure` in top right corner\n   2. Click `Add new Token`\n   3. Get it and paste to config to `JENKINS_API_TOKEN` param\n##### Database Initialization\n\nYou can initialize a scylla cluster in any way you like, either using docker image with docker-compose or using cassandra cluster manager. You will need to create the keyspace manually before you can sync database models.\n\nCreate keyspace according to your configuration.\ne.g. (need to test if it works with RF=1 if not, make it 3)\n```\nCREATE KEYSPACE argus WITH replication = {\'class\': \'SimpleStrategy\', \'replication_factor\' : 1}\n```\n\nInitial sync can be done as follows:\n\n```py\nfrom argus.backend.db import ScyllaCluster\nfrom argus.db.testrun import TestRun\ndb = ScyllaCluster.get()\n\ndb.sync_models() # Syncronizes Object Mapper models\nTestRun.init_own_table() # Syncronizes TestRun table (separate from python-driver Object Mapper)\n\n```\n\nYou can also use `flask sync-models` afterwards during development when making small changes to models.\n\nIt is recommended to set up jenkins api key and run `flask scan-jenkins` afterwards to get basic release/group/test structure.\n\nThere are scripts in `./scripts` directory that can be used to download data from production, upload them into your dev db and fix their relations to other models in your instance of the application. Specifically, `download_runs_from_prod.py` requires additional config, `argus.local.prod.yaml` which is the config used to connect to the production cluster. The scripts are split to prevent mistakes and accidentally affecting production cluster.\n\n##### Configuration\n\nCreate a `argus.local.yaml` configuration file (used to configure database connection) and a `argus_web.yaml` (used for webapp secrets) in your application install directory.\n\n```bash\ncp argus_web.example.yaml argus_web.yaml\ncp argus.yaml argus.local.yaml\n```\n\nOpen `argus.local.yaml` and add the database connection information (contact_points, user, password and keyspace name).\n\nOpen `argus_web.yaml` and change the `SECRET_KEY` value to something secure, like a sha512 digest of random bytes. Fill out GITHUB_* and JENKINS_* variables with their respective values.\n\nRun the application from CLI using:\n\n```bash\nFLASK_ENV="development" FLASK_APP="argus_backend:start_server" FLASK_DEBUG=1 CQLENG_ALLOW_SCHEMA_MANAGEMENT=1 flask run\n```\n\nOmit `FLASK_DEBUG` if running your own debugger (pdb, pycharm, vscode)\n',
-    'author': 'Alexey Kartashov',
-    'author_email': 'alexey.kartashov@scylladb.com',
-    'maintainer': 'None',
-    'maintainer_email': 'None',
-    'url': 'https://github.com/scylladb/argus',
-    'packages': packages,
-    'package_data': package_data,
-    'install_requires': install_requires,
-    'python_requires': '>=3.10,<4.0',
-}
-
-
-setup(**setup_kwargs)