argus-alm 0.14.2__py3-none-any.whl → 0.15.1__py3-none-any.whl

argus/backend/tests/client_service/test_submit_results.py

@@ -0,0 +1,79 @@
+from dataclasses import asdict, dataclass
+from typing import Any
+
+import pytest
+
+from argus.backend.error_handlers import DataValidationError
+from argus.backend.tests.conftest import get_fake_test_run
+from argus.client.generic_result import ColumnMetadata, ResultType, ValidationRule, Status, \
+    StaticGenericResultTable
+
+
+@dataclass
+class SampleCell:
+    column: str
+    row: str
+    value: Any
+    status: Status = Status.UNSET
+
+
+class SampleTable(StaticGenericResultTable):
+    class Meta:
+        name = "Test Table"
+        description = "Test Table Description"
+        sut_package_name = "test_package"
+        Columns = [
+            ColumnMetadata(name="metric1", unit="ms", type=ResultType.FLOAT, higher_is_better=False),
+            ColumnMetadata(name="metric2", unit="ms", type=ResultType.INTEGER, higher_is_better=False),
+        ]
+        ValidationRules = {
+            "metric1": ValidationRule(fixed_limit=100),
+            "metric2": ValidationRule(fixed_limit=200),
+        }
+
+
+def test_submit_results_responds_ok_if_all_cells_pass(fake_test, client_service):
+    run_type, run = get_fake_test_run(test=fake_test)
+    results = SampleTable()
+    results.sut_timestamp = 123
+    sample_data = [
+        SampleCell(column="metric1", row="row1", value=99.99),
+        SampleCell(column="metric2", row="row1", value=199.99),
+    ]
+    for cell in sample_data:
+        results.add_result(column=cell.column, row=cell.row, value=cell.value, status=cell.status)
+    client_service.submit_run(run_type, asdict(run))
+    response = client_service.submit_results(run_type, run.run_id, results.as_dict())
+    assert results.as_dict()["meta"]["sut_package_name"] == "test_package"
+    assert response["status"] == "ok"
+    assert response["message"] == "Results submitted"
+
+
+def test_submit_results_responds_with_error_when_cell_fails_validation(fake_test, client_service):
+    run_type, run = get_fake_test_run(test=fake_test)
+    results = SampleTable()
+    results.sut_timestamp = 123
+    sample_data = [
+        SampleCell(column="metric1", row="row1", value=100.01),  # Exceeds fixed_limit
+        SampleCell(column="metric2", row="row1", value=50),
+    ]
+    for cell in sample_data:
+        results.add_result(column=cell.column, row=cell.row, value=cell.value, status=cell.status)
+    client_service.submit_run(run_type, asdict(run))
+    with pytest.raises(DataValidationError):
+        client_service.submit_results(run_type, run.run_id, results.as_dict())
+
+
+def test_submit_results_responds_with_error_when_cell_has_error(fake_test, client_service):
+    run_type, run = get_fake_test_run(test=fake_test)
+    results = SampleTable()
+    results.sut_timestamp = 123
+    sample_data = [
+        SampleCell(column="metric1", row="row1", value=88, status=Status.ERROR),  # hardcoded error
+        SampleCell(column="metric2", row="row1", value=50),
+    ]
+    for cell in sample_data:
+        results.add_result(column=cell.column, row=cell.row, value=cell.value, status=cell.status)
+    client_service.submit_run(run_type, asdict(run))
+    with pytest.raises(DataValidationError):
+        client_service.submit_results(run_type, run.run_id, results.as_dict())

argus/backend/tests/conftest.py

@@ -0,0 +1,180 @@
+import os
+import time
+import uuid
+from unittest.mock import patch
+
+from cassandra.auth import PlainTextAuthProvider
+from docker import DockerClient
+from flask import g
+
+from argus.backend.util.config import Config
+
+os.environ['DOCKER_HOST'] = ""
+from _pytest.fixtures import fixture
+from docker.errors import NotFound
+from argus.backend.cli import sync_models
+from argus.backend.db import ScyllaCluster
+from argus.backend.models.web import ArgusTest, ArgusGroup, ArgusRelease, User, UserRoles
+from argus.backend.plugins.sct.testrun import SCTTestRunSubmissionRequest
+from argus.backend.service.client_service import ClientService
+from argus.backend.service.release_manager import ReleaseManagerService
+from argus.backend.service.results_service import ResultsService
+import logging
+from cassandra.cluster import Cluster
+
+logging.getLogger().setLevel(logging.INFO)
+os.environ['CQLENG_ALLOW_SCHEMA_MANAGEMENT'] = '1'
+logging.getLogger('cassandra').setLevel(logging.WARNING)
+logging.getLogger('cassandra.connection').setLevel(logging.WARNING)
+logging.getLogger('cassandra.pool').setLevel(logging.WARNING)
+logging.getLogger('cassandra.cluster').setLevel(logging.WARNING)
+
+
+@fixture(scope='session')
+def argus_db():
+    container_name = "argus_test_scylla"
+    docker_client = DockerClient.from_env()
+    need_sync_models = True
+    try:
+        container = docker_client.containers.get(container_name)
+        if container.status != 'running':
+            container.start()
+            print(f"Started existing container '{container_name}'.")
+        else:
+            print(f"Using already running container '{container_name}'.")
+        need_sync_models = False
+    except NotFound:
+        container = docker_client.containers.run(
+            "scylladb/scylla-enterprise:2024.1.10",
+            name=container_name,
+            detach=True,
+            ports={'9042/tcp': 9042},
+            command=[
+                "--smp", "1",
+                "--overprovisioned", "1",
+                "--skip-wait-for-gossip-to-settle", "0",
+                "--endpoint-snitch=SimpleSnitch",
+                "--authenticator", "PasswordAuthenticator",
+            ],
+        )
+        log_wait_timeout = 120
+        start_time = time.time()
+
+        print("Waiting for 'init - serving' message in container logs...")
+        for log in container.logs(stream=True):
+            log_line = log.decode('utf-8')
+            if "init - serving" in log_line:
+                print("'init - serving' message found.")
+                break
+            if "FATAL state" in log_line:
+                raise Exception("ScyllaDB exited unexpectedly. Check container logs for more information.")
+            if time.time() - start_time > log_wait_timeout:
+                raise Exception("ScyllaDB did not log 'init - serving' within the timeout period.")
+
+    container.reload()
+    container_ip = container.attrs['NetworkSettings']['Networks']['bridge']['IPAddress']
+    print(f"Container IP: {container_ip}")
+
+    if need_sync_models:
+        auth_provider = PlainTextAuthProvider(username='cassandra', password='cassandra')
+        cluster = Cluster([container_ip], port=9042, auth_provider=auth_provider)  # Use container IP
+        session = cluster.connect()
+        session.execute("""
+             CREATE KEYSPACE IF NOT EXISTS test_argus
+             WITH replication = {'class': 'SimpleStrategy', 'replication_factor' : 1};
+         """)
+    config = {"SCYLLA_KEYSPACE_NAME": "test_argus", "SCYLLA_CONTACT_POINTS": [container_ip],
+              "SCYLLA_USERNAME": "cassandra", "SCYLLA_PASSWORD": "cassandra", "APP_LOG_LEVEL": "INFO",
+              "EMAIL_SENDER": "unit tester", "EMAIL_SENDER_PASS": "pass", "EMAIL_SENDER_USER": "qa",
+              "EMAIL_SERVER": "fake", "EMAIL_SERVER_PORT": 25}
+    Config.CONFIG = config  # patch config for whole test to avoid using Config.load_yaml_config() required by app context
+    database = ScyllaCluster.get(config)
+    if need_sync_models:
+        sync_models()
+
+    yield database
+    database.shutdown()
+
+
+@fixture(scope='session')
+def argus_app():
+    with patch('argus.backend.service.user.load_logged_in_user') as mock_load:
+        mock_load.return_value = None  # Make the function do nothing so test can override user
+        from argus_backend import argus_app
+        yield argus_app
+
+
+@fixture(scope='session', autouse=True)
+def app_context(argus_db, argus_app):
+    with argus_app.app_context():
+        g.user = User(id=uuid.uuid4(), username='test_user', full_name='Test User',
+                      email="tester@scylladb.com",
+                      roles=[UserRoles.User, UserRoles.Admin, UserRoles.Manager])
+        yield
+
+
+@fixture(scope='session')
+def flask_client(argus_app):
+    return argus_app.test_client()
+
+
+@fixture(scope='session')
+def release_manager_service(argus_db) -> ReleaseManagerService:
+    return ReleaseManagerService()
+
+
+@fixture(scope='session')
+def client_service(argus_db):
+    return ClientService()
+
+
+@fixture(scope='session')
+def results_service(argus_db):
+    return ResultsService()
+
+
+def get_fake_test_run(
+        test: ArgusTest,
+        schema_version: str = "1.0.0",
+        job_url: str = "http://example.com",
+        started_by: str = "default_user",
+        commit_id: str = "default_commit_id",
+        sct_config: dict | None = None,
+        origin_url: str | None = None,
+        branch_name: str | None = "main",
+        runner_public_ip: str | None = None,
+        runner_private_ip: str | None = None
+) -> tuple[str, SCTTestRunSubmissionRequest]:
+    run_id = str(uuid.uuid4())
+    return "scylla-cluster-tests", SCTTestRunSubmissionRequest(
+        schema_version=schema_version,
+        run_id=run_id,
+        job_name=test.build_system_id,
+        job_url=job_url,
+        started_by=started_by,
+        commit_id=commit_id,
+        sct_config=sct_config,
+        origin_url=origin_url,
+        branch_name=branch_name,
+        runner_public_ip=runner_public_ip,
+        runner_private_ip=runner_private_ip
+    )
+
+
+@fixture(scope='session')
+def release(release_manager_service) -> ArgusRelease:
+    name = f"best_results_{time.time_ns()}"
+    return release_manager_service.create_release(name, name, False)
+
+
+@fixture(scope='session')
+def group(release_manager_service, release) -> ArgusGroup:
+    name = f"br_group{time.time_ns()}"
+    return release_manager_service.create_group(name, name, build_system_id=release.name, release_id=str(release.id))
+
+
+@fixture
+def fake_test(release_manager_service, group: ArgusGroup, release: ArgusRelease) -> ArgusTest:
+    name = f"test_{time.time_ns()}"
+    return release_manager_service.create_test(name, name, name, name,
+                                               group_id=str(group.id), release_id=str(release.id), plugin_name='scylla-cluster-tests')

argus/backend/tests/results_service/test_best_results.py

@@ -0,0 +1,178 @@
+import logging
+from dataclasses import asdict, dataclass
+from typing import Any
+
+import pytest
+
+from argus.backend.error_handlers import DataValidationError
+from argus.backend.plugins.sct.testrun import SCTTestRun
+from argus.backend.tests.conftest import get_fake_test_run, fake_test
+from argus.common.enums import TestInvestigationStatus
+from argus.client.generic_result import ColumnMetadata, ResultType, ValidationRule, Status, StaticGenericResultTable
+
+LOGGER = logging.getLogger(__name__)
+
+
+class SampleTable(StaticGenericResultTable):
+    class Meta:
+        name = "Test Table Name"
+        description = "Test Table Description"
+        Columns = [ColumnMetadata(name="h_is_better", unit="ms", type=ResultType.FLOAT, higher_is_better=True),
+                   ColumnMetadata(name="l_is_better", unit="ms", type=ResultType.INTEGER, higher_is_better=False),
+                   ColumnMetadata(name="duration col name", unit="s", type=ResultType.DURATION, higher_is_better=False),
+                   ColumnMetadata(name="non tracked col name", unit="", type=ResultType.FLOAT),
+                   ColumnMetadata(name="text col name", unit="", type=ResultType.TEXT),
+                   ]
+        ValidationRules = {"h_is_better": ValidationRule(best_abs=4),
+                           "l_is_better": ValidationRule(best_pct=50, best_abs=5),
+                           "duration col name": ValidationRule(fixed_limit=590)
+                           }
+
+
+@dataclass
+class SampleCell:
+    column: str
+    row: str
+    value: Any
+    status: Status = Status.UNSET
+
+
+def test_can_track_best_result(fake_test, client_service, results_service, release, group):
+    run_type, run = get_fake_test_run(test=fake_test)
+    results = SampleTable()
+    results.sut_timestamp = 123
+    sample_data = [
+        SampleCell(column="h_is_better", row="row", value=10),
+        SampleCell(column="l_is_better", row="row", value=10),
+        SampleCell(column="duration col name", row="row", value=10),
+    ]
+    for cell in sample_data:
+        results.add_result(column=cell.column, row=cell.row, value=cell.value, status=cell.status)
+    client_service.submit_run(run_type, asdict(run))
+    client_service.submit_results(run_type, run.run_id, results.as_dict())
+    best_results = results_service.get_best_results(fake_test.id, results.name)
+    # all results should be tracked as best - first submission
+    for cell in sample_data:
+        key = f"{cell.column}:{cell.row}"
+        assert best_results[key][-1].value == cell.value
+        assert str(best_results[key][-1].run_id) == run.run_id
+    result_date_h = best_results["h_is_better:row"][-1].result_date  # save the result date for later comparison
+    result_date_duration = best_results["duration col name:row"][-1].result_date
+    # second submission with better results
+    run_type, run = get_fake_test_run(test=fake_test)
+    results = SampleTable()
+    results.sut_timestamp = 124
+    sample_data = [
+        SampleCell(column="h_is_better", row="row", value=15),  # Improved
+        SampleCell(column="l_is_better", row="row", value=5),  # Improved
+        SampleCell(column="duration col name", row="row", value=10),  # Same
+    ]
+    for cell in sample_data:
+        results.add_result(column=cell.column, row=cell.row, value=cell.value, status=cell.status)
+    client_service.submit_run(run_type, asdict(run))
+    client_service.submit_results(run_type, run.run_id, results.as_dict())
+    best_results = results_service.get_best_results(fake_test.id, results.name)
+    # best results should be updated
+    assert best_results["h_is_better:row"][-1].value == 15
+    assert best_results["h_is_better:row"][-1].result_date > result_date_h  # result date should be updated
+    assert best_results["l_is_better:row"][-1].value == 5
+    assert best_results["duration col name:row"][-1].value == 10
+    # result date should not change as was not updated
+    assert best_results["duration col name:row"][-1].result_date == result_date_duration
+
+
+def test_can_enable_best_results_tracking(fake_test, client_service, results_service, release, group):
+    """
+    best results tracking can be enabled by setting higher_is_better in ColumnMetadata to bool value
+    enabling best results tracking for a text column should not break the system
+    """
+    run_type, run = get_fake_test_run(test=fake_test)
+    results = SampleTable()
+    results.sut_timestamp = 123
+    sample_data = [
+        SampleCell(column="h_is_better", row="row", value=10),
+        SampleCell(column="l_is_better", row="row", value=10),
+        SampleCell(column="duration col name", row="row", value=10),
+        SampleCell(column="non tracked col name", row="row", value=10),
+    ]
+    for cell in sample_data:
+        results.add_result(column=cell.column, row=cell.row, value=cell.value, status=cell.status)
+    client_service.submit_run(run_type, asdict(run))
+    client_service.submit_results(run_type, run.run_id, results.as_dict())
+    best_results = results_service.get_best_results(fake_test.id, results.name)
+    assert 'non tracked col name:row' not in best_results  # non tracked column should not be tracked
+
+    class TrackingAllSampleTable(StaticGenericResultTable):
+        class Meta:
+            name = "Test Table Name"
+            description = "Test Table Description"
+            Columns = [ColumnMetadata(name="h_is_better", unit="ms", type=ResultType.FLOAT, higher_is_better=True),
+                       ColumnMetadata(name="l_is_better", unit="ms", type=ResultType.INTEGER, higher_is_better=False),
+                       ColumnMetadata(name="duration col name", unit="s",
+                                      type=ResultType.DURATION, higher_is_better=False),
+                       ColumnMetadata(name="non tracked col name", unit="",
+                                      type=ResultType.FLOAT, higher_is_better=True),
+                       ColumnMetadata(name="text col name", unit="", type=ResultType.TEXT, higher_is_better=True),
+                       ]
+
+        ValidationRules = {"h_is_better": ValidationRule(best_abs=4),
+                           "l_is_better": ValidationRule(best_pct=50, best_abs=5),
+                           "duration col name": ValidationRule(fixed_limit=590)
+                           }
+
+    run_type, run = get_fake_test_run(test=fake_test)
+    results = TrackingAllSampleTable()
+    results.sut_timestamp = 124
+    sample_data = [
+        SampleCell(column="h_is_better", row="row", value=15),  # Improved
+        SampleCell(column="l_is_better", row="row", value=5),  # Improved
+        SampleCell(column="duration col name", row="row", value=10),  # Same
+        SampleCell(column="non tracked col name", row="row", value=10),
+        SampleCell(column="text col name", row="row", value="10"),
+    ]
+    for cell in sample_data:
+        results.add_result(column=cell.column, row=cell.row, value=cell.value, status=cell.status)
+    client_service.submit_run(run_type, asdict(run))
+    client_service.submit_results(run_type, run.run_id, results.as_dict())
+    best_results = results_service.get_best_results(fake_test.id, results.name)
+    assert best_results["h_is_better:row"][-1].value == 15
+    assert best_results["l_is_better:row"][-1].value == 5
+    assert best_results["duration col name:row"][-1].value == 10
+    assert best_results["non tracked col name:row"][-1].value == 10
+    assert 'text col name:row' not in best_results  # text column should not be tracked
+
+
+def test_ignored_runs_are_not_considered_in_best_results(fake_test, client_service, results_service, release, group):
+    run_type, run = get_fake_test_run(test=fake_test)
+    results = SampleTable()
+    results.sut_timestamp = 123
+    sample_data = [
+        SampleCell(column="h_is_better", row="row", value=100),
+        SampleCell(column="l_is_better", row="row", value=10),
+    ]
+    for cell in sample_data:
+        results.add_result(column=cell.column, row=cell.row, value=cell.value, status=cell.status)
+    client_service.submit_run(run_type, asdict(run))
+    client_service.submit_results(run_type, run.run_id, results.as_dict())
+    run_type, run2 = get_fake_test_run(test=fake_test)
+    sample_data = [
+        SampleCell(column="h_is_better", row="row", value=200),
+        SampleCell(column="l_is_better", row="row", value=5),
+    ]
+    for cell in sample_data:
+        results.add_result(column=cell.column, row=cell.row, value=cell.value, status=cell.status)
+    client_service.submit_run(run_type, asdict(run2))
+    with pytest.raises(DataValidationError):
+        client_service.submit_results(run_type, run2.run_id, results.as_dict())
+
+    # ignore the second run
+    run_model = SCTTestRun.get(id=run2.run_id)
+    run_model.investigation_status = TestInvestigationStatus.IGNORED.value
+    run_model.save()
+
+    best_results = results_service.get_best_results(fake_test.id, results.name)
+
+    assert best_results["h_is_better:row"][-1].value == 100  # should not consider the second run
+    assert str(best_results["h_is_better:row"][-1].run_id) == run.run_id
+    assert best_results["l_is_better:row"][-1].value == 10   # should not consider the second run
+    assert str(best_results["l_is_better:row"][-1].run_id) == run.run_id

argus/backend/tests/results_service/test_cell.py

@@ -0,0 +1,65 @@
+from argus.backend.service.results_service import Cell, ArgusGenericResultMetadata
+
+
+def test_cell_initialization():
+    cell = Cell(column="col1", row="row1", status="UNSET")
+    assert cell.column == "col1"
+    assert cell.row == "row1"
+    assert cell.status == "UNSET"
+    assert cell.value is None
+    assert cell.value_text is None
+
+
+def test_valid_rules_and_better_value_should_pass():
+    cell = Cell(column="col1", row="row1", status="UNSET", value=10)
+    table_metadata = ArgusGenericResultMetadata(
+        validation_rules={"col1": [{"fixed_limit": 5}]},
+        columns_meta=[{"name": "col1", "higher_is_better": True}]
+    )
+    best_results = {}
+    cell.update_cell_status_based_on_rules(table_metadata, best_results)
+    assert cell.status == "PASS"
+
+
+def test_valid_rules_and_worse_value_should_fail():
+    cell = Cell(column="col1", row="row1", status="UNSET", value=3)
+    table_metadata = ArgusGenericResultMetadata(
+        validation_rules={"col1": [{"fixed_limit": 5}]},
+        columns_meta=[{"name": "col1", "higher_is_better": True}]
+    )
+    best_results = {}
+    cell.update_cell_status_based_on_rules(table_metadata, best_results)
+    assert cell.status == "ERROR"
+
+
+def test_no_rules_should_keep_status_unset():
+    cell = Cell(column="col1", row="row1", status="UNSET", value=10)
+    table_metadata = ArgusGenericResultMetadata(
+        validation_rules={},
+        columns_meta=[{"name": "col1", "higher_is_better": True}]
+    )
+    best_results = {}
+    cell.update_cell_status_based_on_rules(table_metadata, best_results)
+    assert cell.status == "UNSET"
+
+
+def test_not_unset_status_should_not_be_validated():
+    cell = Cell(column="col1", row="row1", status="PASS", value=10)
+    table_metadata = ArgusGenericResultMetadata(
+        validation_rules={"col1": [{"fixed_limit": 5}]},
+        columns_meta=[{"name": "col1", "higher_is_better": True}]
+    )
+    best_results = {}
+    cell.update_cell_status_based_on_rules(table_metadata, best_results)
+    assert cell.status == "PASS"
+
+
+def test_higher_is_better_false_should_fail():
+    cell = Cell(column="col1", row="row1", status="UNSET", value=10)
+    table_metadata = ArgusGenericResultMetadata(
+        validation_rules={"col1": [{"fixed_limit": 5}]},
+        columns_meta=[{"name": "col1", "higher_is_better": False}]
+    )
+    best_results = {}
+    cell.update_cell_status_based_on_rules(table_metadata, best_results)
+    assert cell.status == "ERROR"