argus-alm 0.14.2__py3-none-any.whl → 0.15.2__py3-none-any.whl

argus/backend/tests/results_service/test_results_service.py

@@ -0,0 +1,203 @@
+from datetime import datetime, timedelta
+from uuid import uuid4
+
+import pytest
+from argus.backend.models.result import ArgusGenericResultMetadata, ArgusGenericResultData, ColumnMetadata, ArgusGraphView
+from argus.backend.plugins.sct.testrun import SCTTestRun
+from argus.backend.plugins.sct.udt import PackageVersion
+from argus.backend.service.results_service import ResultsService
+
+
+@pytest.fixture
+def setup_data(argus_db):
+    test_id = uuid4()
+    table = ArgusGenericResultMetadata(
+        test_id=test_id,
+        name='Test Table',
+        columns_meta=[
+            ColumnMetadata(name='col1', unit='ms', type='FLOAT', higher_is_better=False)
+        ],
+        rows_meta=['row1'],
+        validation_rules={}
+    )
+    data = [
+        ArgusGenericResultData(
+            test_id=test_id,
+            name=table.name,
+            run_id=uuid4(),
+            column='col1',
+            row='row1',
+            sut_timestamp=datetime.today() - timedelta(days=10),
+            value=100.0,
+            status='UNSET'
+        ).save(),
+        ArgusGenericResultData(
+            test_id=test_id,
+            name=table.name,
+            run_id=uuid4(),
+            column='col1',
+            row='row1',
+            sut_timestamp=datetime.today() - timedelta(days=5),
+            value=150.0,
+            status='UNSET'
+        ).save(),
+        ArgusGenericResultData(
+            test_id=test_id,
+            name=table.name,
+            run_id=uuid4(),
+            column='col1',
+            row='row1',
+            sut_timestamp=datetime.today() - timedelta(days=1),
+            value=200.0,
+            status='UNSET'
+        ).save()
+    ]
+    return test_id, table, data
+
+
+def test_results_service_should_return_results_within_date_range(setup_data):
+    test_id, table, data = setup_data
+    service = ResultsService()
+
+    start_date = datetime.today() - timedelta(days=7)
+    end_date = datetime.today() - timedelta(days=2)
+
+    filtered_data = service._get_tables_data(
+        test_id=test_id,
+        table_name=table.name,
+        ignored_runs=[],
+        start_date=start_date,
+        end_date=end_date
+    )
+
+    assert len(filtered_data) == 1
+    assert filtered_data[0].value == 150.0
+
+
+def test_results_service_should_return_no_results_outside_date_range(setup_data):
+    test_id, table, data = setup_data
+    service = ResultsService()
+
+    start_date = datetime.today() - timedelta(days=20)
+    end_date = datetime.today() - timedelta(days=15)
+
+    filtered_data = service._get_tables_data(
+        test_id=test_id,
+        table_name=table.name,
+        ignored_runs=[],
+        start_date=start_date,
+        end_date=end_date
+    )
+
+    assert len(filtered_data) == 0
+
+
+def test_results_service_should_return_all_results_with_no_date_range(setup_data):
+    test_id, table, data = setup_data
+    service = ResultsService()
+
+    filtered_data = service._get_tables_data(
+        test_id=test_id,
+        table_name=table.name,
+        ignored_runs=[]
+    )
+
+    assert len(filtered_data) == 3
+
+
+def test_get_tests_by_version_groups_runs_correctly(argus_db):
+    test_id1 = uuid4()
+    test_id2 = uuid4()
+    run_id1 = uuid4()
+    run_id2 = uuid4()
+    run_id3 = uuid4()
+    run_id4 = uuid4()
+    pkg_v4_0 = PackageVersion(name='scylla', version='4.0', date='2021-01-01', revision_id='', build_id='')
+    pkg_v4_1 = PackageVersion(name='scylla', version='4.1', date='2021-02-01', revision_id='', build_id='')
+
+    SCTTestRun(
+        id=run_id1,
+        build_id='build_id1',
+        test_id=test_id1,
+        test_method='test_method1',  # Changed to 'test_method'
+        investigation_status='',
+        packages=[pkg_v4_0]
+    ).save()
+    SCTTestRun(
+        id=run_id2,
+        build_id='build_id1',
+        test_id=test_id1,
+        test_method='test_method2',  # Changed to 'test_method'
+        investigation_status='ignored',
+        packages=[pkg_v4_0]
+    ).save()
+    SCTTestRun(
+        id=run_id3,
+        build_id='build_id1',
+        test_id=test_id2,
+        test_method='test_method1',  # Changed to 'test_method'
+        investigation_status='',
+        packages=[pkg_v4_0]
+    ).save()
+    SCTTestRun(
+        id=run_id4,
+        build_id='build_id1',
+        test_id=test_id2,
+        test_method='test_method1',  # Changed to 'test_method'
+        investigation_status='',
+        packages=[pkg_v4_1]
+    ).save()
+
+    sut_package_name = 'scylla'
+    test_ids = [test_id1, test_id2]
+    service = ResultsService()
+    service._exclude_disabled_tests = lambda x: x
+    result = service.get_tests_by_version(sut_package_name, test_ids)
+
+    expected_result = {'test_info': {str(test_id1): {'build_id': 'build_id1',
+                                                     'name': None},
+                                     str(test_id2): {'build_id': 'build_id1',
+                                                     'name': None}},
+                       'versions': {'4.0-2021-01-01-': {
+                           str(test_id1): {'test_method1': {'run_id': str(run_id1),
+                                                            'started_by': None,
+                                                            'status': 'created'}},
+                           str(test_id2): {'test_method1': {'run_id': str(run_id3),
+                                                            'started_by': None,
+                                                            'status': 'created'}}},
+                           '4.1-2021-02-01-': {str(test_id2): {
+                               'test_method1': {'run_id': str(run_id4),
+                                                'started_by': None,
+                                                'status': 'created'}}}}}
+    assert result == expected_result
+
+
+def test_create_update_argus_graph_view_should_create() -> None:
+    service = ResultsService()
+    test_id = uuid4()
+    service.create_argus_graph_view(test_id, "MyView", "MyDescription")
+    result = service.get_argus_graph_views(test_id)[0]
+    assert result is not None
+    assert result.name == "MyView"
+    assert result.description == "MyDescription"
+    assert result.graphs == {}
+
+
+def test_create_update_argus_graph_view_should_update() -> None:
+    service = ResultsService()
+    test_id = uuid4()
+    graph_view = service.create_argus_graph_view(test_id, "OldName", "OldDesc")
+    service.update_argus_graph_view(test_id, graph_view.id, "NewName", "NewDesc", {"graph2": "new_data"})
+    updated = service.get_argus_graph_views(test_id)[0]
+    assert updated.name == "NewName"
+    assert updated.description == "NewDesc"
+    assert updated.graphs == {"graph2": "new_data"}
+
+
+def test_get_argus_graph_views_should_return_list() -> None:
+    service = ResultsService()
+    test_id = uuid4()
+    service.create_argus_graph_view(test_id, "View1", "Desc1")
+    service.create_argus_graph_view(test_id, "View2", "Desc2")
+    views = service.get_argus_graph_views(test_id)
+    assert len(views) == 2

argus/backend/tests/results_service/test_validation_rules.py

@@ -0,0 +1,213 @@
+import logging
+from dataclasses import asdict, dataclass
+from typing import Any
+from uuid import UUID
+
+import pytest
+
+from argus.backend.error_handlers import DataValidationError
+from argus.backend.tests.conftest import get_fake_test_run, fake_test
+from argus.client.generic_result import ColumnMetadata, ResultType, ValidationRule, Status, StaticGenericResultTable
+
+LOGGER = logging.getLogger(__name__)
+
+
+class SampleTable(StaticGenericResultTable):
+    class Meta:
+        name = "Test Table Name"
+        description = "Test Table Description"
+        Columns = [ColumnMetadata(name="h_is_better", unit="ms", type=ResultType.FLOAT, higher_is_better=True),
+                   ColumnMetadata(name="l_is_better", unit="ms", type=ResultType.INTEGER, higher_is_better=False),
+                   ColumnMetadata(name="duration col name", unit="s", type=ResultType.DURATION, higher_is_better=False),
+                   ColumnMetadata(name="non tracked col name", unit="", type=ResultType.FLOAT),
+                   ColumnMetadata(name="text col name", unit="", type=ResultType.TEXT),
+                   ]
+        ValidationRules = {"h_is_better": ValidationRule(best_abs=4),
+                           "l_is_better": ValidationRule(best_pct=50),
+                           "duration col name": ValidationRule(fixed_limit=590)
+                           }
+
+
+@dataclass
+class SampleCell:
+    column: str
+    row: str
+    value: Any
+    status: Status = Status.UNSET
+
+
+def results_to_dict(results):
+    actual_cells = {}
+    table_data = results['Test Table Name']['table_data']
+
+    for row_key, row_data in table_data.items():
+        for col_name, col_data in row_data.items():
+            if col_name not in actual_cells:
+                actual_cells[col_name] = {}
+            actual_cells[col_name][row_key] = {
+                'value': col_data['value'],
+                'status': col_data['status']
+            }
+    return actual_cells
+
+
+def test_can_track_validation_rules_changes(fake_test, client_service, results_service, release, group):
+    run_type, run = get_fake_test_run(test=fake_test)
+    results = SampleTable()
+    results.sut_timestamp = 123
+    sample_data = [
+        SampleCell(column="h_is_better", row="row", value=10),
+        SampleCell(column="l_is_better", row="row", value=100),
+        SampleCell(column="duration col name", row="row", value=10),
+        SampleCell(column="non tracked col name", row="row", value=10),
+        SampleCell(column="text col name", row="row", value="a value"),
+    ]
+    for cell in sample_data:
+        results.add_result(column=cell.column, row=cell.row, value=cell.value, status=cell.status)
+    client_service.submit_run(run_type, asdict(run))
+    client_service.submit_results(run_type, run.run_id, results.as_dict())
+    run_results = results_service.get_run_results(fake_test.id, UUID(run.run_id))
+    actual_cells = results_to_dict(run_results[0])
+    # all results should be marked as passed
+    for cell in sample_data:
+        if cell.column == "text col name" or cell.column == "non tracked col name":
+            assert actual_cells[cell.column][cell.row]['value'] == cell.value
+            assert actual_cells[cell.column][cell.row]['status'] == "UNSET"
+        else:
+            assert actual_cells[cell.column][cell.row]['value'] == cell.value
+            assert actual_cells[cell.column][cell.row]['status'] == "PASS"
+
+    # test validation rules are applied correctly
+    run_type, run = get_fake_test_run(test=fake_test)
+    results = SampleTable()
+    results.sut_timestamp = 124
+    sample_data = [
+        SampleCell(column="h_is_better", row="row", value=6),
+        SampleCell(column="l_is_better", row="row", value=150),
+        SampleCell(column="duration col name", row="row", value=600),
+        SampleCell(column="non tracked col name", row="row", value=12),
+        SampleCell(column="text col name", row="row", value="a value"),
+    ]
+    for cell in sample_data:
+        results.add_result(column=cell.column, row=cell.row, value=cell.value, status=cell.status)
+    client_service.submit_run(run_type, asdict(run))
+    with pytest.raises(DataValidationError):
+        client_service.submit_results(run_type, run.run_id, results.as_dict())
+    run_results = results_service.get_run_results(fake_test.id, UUID(run.run_id))
+    actual_cells = results_to_dict(run_results[0])
+    for cell in sample_data:
+        if cell.column == "text col name" or cell.column == "non tracked col name":
+            assert actual_cells[cell.column][cell.row][
+                'value'] == cell.value, f"Expected {cell.value} but got {actual_cells[cell.column][cell.row]['value']}"
+            assert actual_cells[cell.column][cell.row][
+                'status'] == "UNSET", f"Expected ERROR for {cell.column} but got {actual_cells[cell.column][cell.row]['status']}"
+        else:
+            assert actual_cells[cell.column][cell.row][
+                'value'] == cell.value, f"Expected {cell.value} but got {actual_cells[cell.column][cell.row]['value']}"
+            assert actual_cells[cell.column][cell.row][
+                'status'] == "ERROR", f"Expected ERROR for {cell.column} but got {actual_cells[cell.column][cell.row]['status']}"
+
+    # new best result appears
+    run_type, run = get_fake_test_run(test=fake_test)
+    results = SampleTable()
+    results.sut_timestamp = 125
+    sample_data = [
+        SampleCell(column="h_is_better", row="row", value=100),
+        SampleCell(column="l_is_better", row="row", value=50),
+        SampleCell(column="duration col name", row="row", value=10),
+        SampleCell(column="non tracked col name", row="row", value=12),
+        SampleCell(column="text col name", row="row", value="a value"),
+    ]
+    for cell in sample_data:
+        results.add_result(column=cell.column, row=cell.row, value=cell.value, status=cell.status)
+    client_service.submit_run(run_type, asdict(run))
+    client_service.submit_results(run_type, run.run_id, results.as_dict())
+    run_results = results_service.get_run_results(fake_test.id, UUID(run.run_id))
+    actual_cells = results_to_dict(run_results[0])
+    for cell in sample_data:
+        if cell.column == "text col name" or cell.column == "non tracked col name":
+            assert actual_cells[cell.column][cell.row][
+                'value'] == cell.value, f"Expected {cell.value} but got {actual_cells[cell.column][cell.row]['value']}"
+            assert actual_cells[cell.column][cell.row][
+                'status'] == "UNSET", f"Expected PASS for {cell.column} but got {actual_cells[cell.column][cell.row]['status']}"
+        else:
+            assert actual_cells[cell.column][cell.row][
+                'value'] == cell.value, f"Expected {cell.value} but got {actual_cells[cell.column][cell.row]['value']}"
+            assert actual_cells[cell.column][cell.row][
+                'status'] == "PASS", f"Expected PASS for {cell.column} but got {actual_cells[cell.column][cell.row]['status']}"
+
+    # validation should be now with new best results
+    run_type, run = get_fake_test_run(test=fake_test)
+    results = SampleTable()
+    results.sut_timestamp = 126
+    sample_data = [
+        SampleCell(column="h_is_better", row="row", value=95),
+        SampleCell(column="l_is_better", row="row", value=75),
+        SampleCell(column="duration col name", row="row", value=590),
+        SampleCell(column="non tracked col name", row="row", value=12),
+        SampleCell(column="text col name", row="row", value="a value"),
+    ]
+    for cell in sample_data:
+        results.add_result(column=cell.column, row=cell.row, value=cell.value, status=cell.status)
+    client_service.submit_run(run_type, asdict(run))
+    with pytest.raises(DataValidationError):
+        client_service.submit_results(run_type, run.run_id, results.as_dict())
+    run_results = results_service.get_run_results(fake_test.id, UUID(run.run_id))
+    actual_cells = results_to_dict(run_results[0])
+    for cell in sample_data:
+        if cell.column == "text col name" or cell.column == "non tracked col name":
+            assert actual_cells[cell.column][cell.row][
+                'value'] == cell.value, f"Expected {cell.value} but got {actual_cells[cell.column][cell.row]['value']}"
+            assert actual_cells[cell.column][cell.row][
+                'status'] == "UNSET", f"Expected ERROR for {cell.column} but got {actual_cells[cell.column][cell.row]['status']}"
+        else:
+            assert actual_cells[cell.column][cell.row][
+                'value'] == cell.value, f"Expected {cell.value} but got {actual_cells[cell.column][cell.row]['value']}"
+            assert actual_cells[cell.column][cell.row][
+                'status'] == "ERROR", f"Expected ERROR for {cell.column} but got {actual_cells[cell.column][cell.row]['status']}"
+
+    # applying new validation rules should be taken into account for next results, new rules are less strict
+    # duration col name rule is removed and 'non tracked col name' has new fixed limit
+    class NewRulesSampleTable(SampleTable):
+        class Meta:
+            name = "Test Table Name"
+            description = "Test Table Description"
+            Columns = [ColumnMetadata(name="h_is_better", unit="ms", type=ResultType.FLOAT, higher_is_better=True),
+                       ColumnMetadata(name="l_is_better", unit="ms", type=ResultType.INTEGER, higher_is_better=False),
+                       ColumnMetadata(name="duration col name", unit="s",
+                                      type=ResultType.DURATION, higher_is_better=False),
+                       ColumnMetadata(name="non tracked col name", unit="",
+                                      type=ResultType.FLOAT, higher_is_better=True),
+                       ColumnMetadata(name="text col name", unit="", type=ResultType.TEXT),
+                       ]
+            ValidationRules = {"h_is_better": ValidationRule(best_abs=100),
+                               "l_is_better": ValidationRule(best_pct=90),
+                               "non tracked col name": ValidationRule(fixed_limit=100)  # new rule, removed old one too
+                               }
+    run_type, run = get_fake_test_run(test=fake_test)
+    results = NewRulesSampleTable()
+    results.sut_timestamp = 122
+    sample_data = [
+        SampleCell(column="h_is_better", row="row", value=95),
+        SampleCell(column="l_is_better", row="row", value=75),
+        SampleCell(column="duration col name", row="row", value=691),
+        SampleCell(column="non tracked col name", row="row", value=101),
+        SampleCell(column="text col name", row="row", value="a value"),
+    ]
+    for cell in sample_data:
+        results.add_result(column=cell.column, row=cell.row, value=cell.value, status=cell.status)
+    client_service.submit_run(run_type, asdict(run))
+    client_service.submit_results(run_type, run.run_id, results.as_dict())
+    run_results = results_service.get_run_results(fake_test.id, UUID(run.run_id))
+    actual_cells = results_to_dict(run_results[0])
+    for cell in sample_data:
+        if cell.column == "text col name":
+            assert actual_cells[cell.column][cell.row][
+                'value'] == cell.value, f"Expected {cell.value} but got {actual_cells[cell.column][cell.row]['value']}"
+            assert actual_cells[cell.column][cell.row][
+                'status'] == "UNSET", f"Expected UNSET for {cell.column} but got {actual_cells[cell.column][cell.row]['status']}"
+        else:
+            assert actual_cells[cell.column][cell.row][
+                'value'] == cell.value, f"Expected {cell.value} but got {actual_cells[cell.column][cell.row]['value']}"
+            assert actual_cells[cell.column][cell.row][
+                'status'] == "PASS", f"Expected PASS for {cell.column} but got {actual_cells[cell.column][cell.row]['status']}"