argus-alm 0.12.7__tar.gz → 0.12.9__tar.gz

{argus_alm-0.12.7 → argus_alm-0.12.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: argus-alm
-Version: 0.12.7
+Version: 0.12.9
 Summary: Argus
 Home-page: https://github.com/scylladb/argus
 License: Apache-2.0

{argus_alm-0.12.7 → argus_alm-0.12.9}/argus/backend/cli.py

@@ -15,6 +15,9 @@ LOGGER = logging.getLogger(__name__)
 @click.command('sync-models')
 @with_appcontext
 def sync_models_command():
+    sync_models()
+
+def sync_models():
     cluster = ScyllaCluster.get()
     cluster.sync_core_tables()
     LOGGER.info("Synchronizing plugin types...")
@@ -29,7 +32,6 @@ def sync_models_command():
     LOGGER.info("Plugins ready.")
     click.echo("All models synchronized.")
 
-
 @cli_bp.cli.add_command
 @click.command('scan-jenkins')
 @with_appcontext

{argus_alm-0.12.7 → argus_alm-0.12.9}/argus/backend/controller/api.py

@@ -4,7 +4,7 @@ import requests
 from flask import (
     Blueprint,
     g,
-    request
+    request, Response
 )
 from flask.json import jsonify
 from argus.backend.error_handlers import handle_api_exception
@@ -382,18 +382,21 @@ def test_info():
         "response": info
     }
 
-@bp.route("/test-results", methods=["GET"])
+@bp.route("/test-results", methods=["GET", "HEAD"])
 @api_login_required
 def test_results():
     test_id = request.args.get("testId")
     if not test_id:
         raise Exception("No testId provided")
     service = ResultsService()
-    info = service.get_results(test_id=UUID(test_id))
+    if request.method == 'HEAD':
+        exists = service.is_results_exist(test_id=UUID(test_id))
+        return Response(status=200 if exists else 404)
+    graphs, ticks = service.get_test_graphs(test_id=UUID(test_id))
 
     return {
         "status": "ok",
-        "response": info
+        "response": {"graphs": graphs, "ticks": ticks}
     }
 
 @bp.route("/test_run/comment/get", methods=["GET"])  # TODO: remove

{argus_alm-0.12.7 → argus_alm-0.12.9}/argus/backend/controller/client_api.py

@@ -105,8 +105,4 @@ def run_finalize(run_type: str, run_id: str):
 @api_login_required
 def submit_results(run_type: str, run_id: str):
     payload = get_payload(request)
-    result = ClientService().submit_results(run_type=run_type, run_id=run_id, results=payload)
-    return {
-        "status": "ok",
-        "response": result
-    }
+    return ClientService().submit_results(run_type=run_type, run_id=run_id, results=payload)

{argus_alm-0.12.7 → argus_alm-0.12.9}/argus/backend/controller/testrun_api.py

@@ -8,6 +8,7 @@ from flask import (
 from argus.backend.error_handlers import handle_api_exception
 from argus.backend.models.web import ArgusTest
 from argus.backend.service.jenkins_service import JenkinsService
+from argus.backend.service.results_service import ResultsService
 from argus.backend.service.testrun import TestRunService
 from argus.backend.service.user import api_login_required
 from argus.backend.util.common import get_payload
@@ -67,7 +68,7 @@ def test_run_activity(run_id: str):
 @bp.route("/run/<string:test_id>/<string:run_id>/fetch_results", methods=["GET"])
 @api_login_required
 def fetch_results(test_id: str, run_id: str):
-    tables = TestRunService().fetch_results(test_id=UUID(test_id), run_id=UUID(run_id))
+    tables = ResultsService().get_run_results(test_id=UUID(test_id), run_id=UUID(run_id))
     return {
         "status": "ok",
         "tables": tables

{argus_alm-0.12.7 → argus_alm-0.12.9}/argus/backend/db.py

@@ -54,7 +54,7 @@ class ScyllaCluster:
         return self.cluster.connect(keyspace=self.config["SCYLLA_KEYSPACE_NAME"])
 
     @classmethod
-    def get(cls, config: Config = None) -> 'ScyllaCluster':
+    def get(cls, config: dict = None) -> 'ScyllaCluster':
         if cls.APP_INSTANCE:
             return cls.APP_INSTANCE
 

argus_alm-0.12.9/argus/backend/models/result.py

@@ -0,0 +1,138 @@
+import math
+from datetime import datetime, timezone
+
+from cassandra.cqlengine import columns
+from cassandra.cqlengine.models import Model
+from cassandra.cqlengine.usertype import UserType
+
+class ValidationRules(UserType):
+    valid_from = columns.DateTime()
+    best_pct = columns.Double()  # max value limit relative to best result in percent unit
+    best_abs = columns.Double()  # max value limit relative to best result in absolute unit
+    fixed_limit = columns.Double()  # fixed limit
+
+class ColumnMetadata(UserType):
+    name = columns.Ascii()
+    unit = columns.Text()
+    type = columns.Ascii()
+    higher_is_better = columns.Boolean()  # used for tracking best results, if None - no tracking
+
+
+class ArgusGenericResultMetadata(Model):
+    __table_name__ = "generic_result_metadata_v1"
+    test_id = columns.UUID(partition_key=True)
+    name = columns.Text(required=True, primary_key=True)
+    description = columns.Text()
+    columns_meta = columns.List(value_type=columns.UserDefinedType(ColumnMetadata))
+    validation_rules = columns.Map(key_type=columns.Ascii(), value_type=columns.List(columns.UserDefinedType(ValidationRules)))
+    rows_meta = columns.List(value_type=columns.Ascii())
+
+    def __init__(self, **kwargs):
+        kwargs["columns_meta"] = [ColumnMetadata(**col) for col in kwargs.pop('columns_meta', [])]
+        validation_rules = kwargs.pop('validation_rules', {})
+
+        if validation_rules:
+            for column, rule in validation_rules.items():
+                if not isinstance(rule, list):
+                    rule['valid_from'] = datetime.now(timezone.utc)
+                    validation_rules[column] = [rule]
+            kwargs["validation_rules"] = {k: [ValidationRules(**rules) for rules in v] for k, v in validation_rules.items()}
+        super().__init__(**kwargs)
+
+    def update_validation_rules(self, key: str, new_rule_dict: dict) -> bool:
+        """
+        Checks if the most recent ValidationRule for the given key matches the new_rule_dict.
+        If not, adds the new rule to the list with the current timestamp.
+
+        :param key: The key (column name) in the validation_rules map to update.
+        :param new_rule_dict: A dictionary containing the new validation rule values.
+        :return: True if a new rule was added, False if the existing rule matches.
+        """
+        rules_list = self.validation_rules.get(key, [])
+        most_recent_rule = None
+
+        if rules_list:
+            most_recent_rule = rules_list[-1]
+
+        fields_to_compare = [field for field in ValidationRules._fields if field != 'valid_from']
+        rules_match = True
+        if most_recent_rule:
+            for field in fields_to_compare:
+                db_value = getattr(most_recent_rule, field)
+                new_value = new_rule_dict.get(field)
+                if db_value is None and new_value is None:
+                    continue
+                if db_value is None or new_value is None:
+                    rules_match = False
+                    break
+                if not math.isclose(db_value, new_value, rel_tol=1e-9, abs_tol=0.0):
+                    rules_match = False
+                    break
+        else:
+            rules_match = False
+
+        if not rules_match:
+            new_rule = ValidationRules(
+                valid_from=datetime.now(timezone.utc),
+                best_pct=new_rule_dict.get('best_pct'),
+                best_abs=new_rule_dict.get('best_abs'),
+                fixed_limit=new_rule_dict.get('fixed_limit')
+            )
+            rules_list.append(new_rule)
+            self.validation_rules = self.validation_rules or {}
+            self.validation_rules.update({key: rules_list})
+            return True
+
+        return False  # Existing rule matches
+
+    def update_if_changed(self, new_data: dict) -> "ArgusGenericResultMetadata":
+        """
+        Updates table metadata if changed column/description or new rows were added.
+        See that rows can only be added, not removed once was sent.
+        Columns may be removed, but data in results table persists.
+        """
+        updated = False
+        for field, value in new_data.items():
+            if field == "columns_meta":
+                value = [ColumnMetadata(**col) for col in value]
+                if self.columns_meta != value:
+                    self.columns_meta = value
+                    updated = True
+            elif field == "rows_meta":
+                added_rows = []
+                for row in value:
+                    if row not in self.rows_meta:
+                        added_rows.append(row)
+                        updated = True
+                self.rows_meta += added_rows
+            elif field == "validation_rules":
+                if any([self.update_validation_rules(key, rules) for key, rules in value.items()]):
+                    updated = True
+            elif getattr(self, field) != value:
+                setattr(self, field, value)
+                updated = True
+
+        if updated:
+            self.save()
+        return self
+
+class ArgusGenericResultData(Model):
+    __table_name__ = "generic_result_data_v1"
+    test_id = columns.UUID(partition_key=True)
+    name = columns.Text(partition_key=True)
+    run_id = columns.UUID(primary_key=True)
+    column = columns.Ascii(primary_key=True, index=True)
+    row = columns.Ascii(primary_key=True, index=True)
+    sut_timestamp = columns.DateTime()  # for sorting
+    value = columns.Double()
+    value_text = columns.Text()
+    status = columns.Ascii()
+
+class ArgusBestResultData(Model):
+    __table_name__ = "generic_result_best_v1"
+    test_id = columns.UUID(partition_key=True)
+    name = columns.Text(partition_key=True)
+    key = columns.Ascii(primary_key=True)  # represents pair column:row
+    result_date = columns.DateTime(primary_key=True, clustering_order="DESC")
+    value = columns.Double()
+    run_id = columns.UUID()

{argus_alm-0.12.7 → argus_alm-0.12.9}/argus/backend/models/web.py

@@ -6,7 +6,7 @@ from cassandra.cqlengine.usertype import UserType
 from cassandra.cqlengine import columns
 from cassandra.util import uuid_from_time, unix_time_from_uuid1  # pylint: disable=no-name-in-module
 
-from argus.backend.models.result import ArgusGenericResultMetadata, ArgusGenericResultData
+from argus.backend.models.result import ArgusGenericResultMetadata, ArgusGenericResultData, ArgusBestResultData
 
 
 def uuid_now():
@@ -381,6 +381,7 @@ USED_MODELS: list[Model] = [
     ArgusScheduleTest,
     ArgusGenericResultMetadata,
     ArgusGenericResultData,
+    ArgusBestResultData,
 ]
 
 USED_TYPES: list[UserType] = [

{argus_alm-0.12.7 → argus_alm-0.12.9}/argus/backend/plugins/loader.py

@@ -30,7 +30,7 @@ def plugin_loader() -> dict[str, PluginInfoBase]:
 
 
 AVAILABLE_PLUGINS = plugin_loader()
-
+print(AVAILABLE_PLUGINS)
 
 def all_plugin_models(include_all=False) -> list[PluginModelBase]:
     return [model for plugin in AVAILABLE_PLUGINS.values() for model in plugin.all_models if issubclass(model, PluginModelBase) or include_all]

{argus_alm-0.12.7 → argus_alm-0.12.9}/argus/backend/plugins/sct/testrun.py

@@ -1,6 +1,6 @@
 from enum import Enum
 import logging
-from datetime import datetime
+from datetime import datetime, timezone
 from dataclasses import dataclass, field
 from typing import Optional
 from uuid import UUID
@@ -254,8 +254,15 @@ class SCTTestRun(PluginModelBase):
     def sut_timestamp(self) -> float:
         """converts scylla-server date to timestamp and adds revision in subseconds precision to diffirentiate
         scylla versions from the same day. It's not perfect, but we don't know exact version time."""
-        scylla_package = [package for package in self.packages if package.name == "scylla-server"][0]
-        return (datetime.strptime(scylla_package.date, '%Y%m%d').timestamp()
+        try:
+            scylla_package_upgraded = [package for package in self.packages if package.name == "scylla-server-upgraded"][0]
+        except IndexError:
+            scylla_package_upgraded = None
+        try:
+            scylla_package = [package for package in self.packages if package.name == "scylla-server"][0]
+        except IndexError:
+            raise ValueError("Scylla package not found in packages - cannot determine SUT timestamp")
+        return (datetime.strptime(scylla_package.date, '%Y%m%d').replace(tzinfo=timezone.utc).timestamp()
                 + int(scylla_package.revision_id, 16) % 1000000 / 1000000)
 
 

{argus_alm-0.12.7 → argus_alm-0.12.9}/argus/backend/service/client_service.py

@@ -1,8 +1,14 @@
+import operator
+from dataclasses import asdict, is_dataclass
+from datetime import datetime, timezone
+from functools import partial
 from uuid import UUID
+
 from argus.backend.db import ScyllaCluster
 from argus.backend.models.result import ArgusGenericResultMetadata, ArgusGenericResultData
 from argus.backend.plugins.core import PluginModelBase
 from argus.backend.plugins.loader import AVAILABLE_PLUGINS
+from argus.backend.service.results_service import ResultsService, Cell
 from argus.backend.util.enums import TestStatus
 
 
@@ -25,8 +31,9 @@ class ClientService:
     def submit_run(self, run_type: str, request_data: dict) -> str:
         model = self.get_model(run_type)
         model.submit_run(request_data=request_data)
+
         return "Created"
-    
+
     def get_run(self, run_type: str, run_id: str):
         model = self.get_model(run_type)
         try:
@@ -79,23 +86,37 @@ class ClientService:
 
         return "Finalized"
 
-    def submit_results(self, run_type: str, run_id: str, results: dict) -> str:
+    def submit_results(self, run_type: str, run_id: str, results: dict) -> dict[str, str]:
         model = self.get_model(run_type)
-        run = model.load_test_run(UUID(run_id))
-        existing_table = ArgusGenericResultMetadata.objects(test_id=run.test_id, name=results["meta"]["name"]).first()
-        if existing_table:
-            existing_table.update_if_changed(results["meta"])
+        try:
+            run = model.load_test_run(UUID(run_id))
+        except model.DoesNotExist:
+            return {"status": "error", "response": {
+                "exception": "DoesNotExist",
+                "arguments": [run_id]
+            }}
+        table_name = results["meta"]["name"]
+        results_service = ResultsService()
+        cells = [Cell(**cell) for cell in results["results"]]
+        table_metadata = results_service.get_table_metadata(test_id=run.test_id, table_name=table_name)
+        if table_metadata:
+            table_metadata = table_metadata.update_if_changed(results["meta"])
         else:
-            ArgusGenericResultMetadata(test_id=run.test_id, **results["meta"]).save()
+            table_metadata = ArgusGenericResultMetadata(test_id=run.test_id, **results["meta"])
+            table_metadata.save()
         if results.get("sut_timestamp", 0) == 0:
             results["sut_timestamp"] = run.sut_timestamp()  # automatic sut_timestamp
+        results["sut_timestamp"] = datetime.fromtimestamp(results["sut_timestamp"])
+        best_results = results_service.update_best_results(test_id=run.test_id, table_name=table_name, table_metadata=table_metadata,
+                                                           cells=cells, run_id=run_id)
         table_name = results["meta"]["name"]
         sut_timestamp = results["sut_timestamp"]
-        for cell in results["results"]:
+        for cell in cells:
+            cell.update_cell_status_based_on_rules(table_metadata, best_results)
             ArgusGenericResultData(test_id=run.test_id,
                                    run_id=run.id,
                                    name=table_name,
                                    sut_timestamp=sut_timestamp,
-                                   **cell
+                                   **asdict(cell)
                                    ).save()
-        return "Submitted"
+        return {"status": "ok", "message": "Results submitted"}

{argus_alm-0.12.7 → argus_alm-0.12.9}/argus/backend/service/notification_manager.py

@@ -93,7 +93,8 @@ class NotificationSenderBase:
 
 class ArgusDBNotificationSaver(NotificationSenderBase):
     CONTENT_TEMPLATES = {
-        ArgusNotificationTypes.Mention: lambda p: render_template("notifications/mention.html.j2", **p if p else {})
+        ArgusNotificationTypes.Mention: lambda p: render_template("notifications/mention.html.j2", **p if p else {}),
+        ArgusNotificationTypes.AssigneeChange: lambda p: render_template("notifications/assigned.html.j2", **p if p else {}),
     }
 
     def send_notification(self, receiver: UUID, sender: UUID, notification_type: ArgusNotificationTypes, source_type: ArgusNotificationSourceTypes,
@@ -117,7 +118,8 @@ class ArgusDBNotificationSaver(NotificationSenderBase):
 class EmailNotificationServiceSender(NotificationSenderBase):
     CONTENT_TEMPLATES = {
         ArgusNotificationTypes.Mention: lambda p: render_template(
-            "notifications/email_mention.html.j2", **p if p else {})
+            "notifications/email_mention.html.j2", **p if p else {}),
+        ArgusNotificationTypes.AssigneeChange: lambda p: render_template("notifications/assigned_email.html.j2", **p if p else {}),
     }
 
     def __init__(self):