argus-alm 0.14.2__py3-none-any.whl → 0.15.2__py3-none-any.whl

argus/backend/util/logsetup.py

@@ -0,0 +1,80 @@
+import logging
+from logging.config import dictConfig
+from flask import has_request_context, request
+
+LOG_FORMAT_REQUEST = "[%(levelcolor)s%(levelname)s%(colorreset)s] %(grey)s<%(remote_addr)s - %(url)s - %(endpoint)s>%(colorreset)s - %(module)s::%(funcName)s - %(message)s"
+
+
+class ArgusRequestLogFormatter(logging.Formatter):
+    yellow = "\x1b[33;20m"
+    red = "\x1b[31;20m"
+    blue = "\x1b[34;20m"
+    bold_red = "\x1b[31;1m"
+    reset = "\x1b[0m"
+    grey = "\x1b[38;2;200;200;200m"
+    color_map = {
+        logging.DEBUG: grey,
+        logging.INFO: blue,
+        logging.WARNING: yellow,
+        logging.ERROR: red,
+        logging.CRITICAL: bold_red
+    }
+
+    def format(self, record: logging.LogRecord) -> str:
+        record.grey = self.grey
+        record.colorreset = self.reset
+        record.levelcolor = self.color_map.get(record.levelno, self.grey)
+        if has_request_context():
+            record.url = request.url
+            record.remote_addr = request.remote_addr
+            record.endpoint = request.endpoint
+        else:
+            record.url = ''
+            record.remote_addr = ''
+            record.endpoint = ''
+        return super().format(record)
+
+
+def setup_application_logging(log_level=logging.INFO):
+    dictConfig({
+        'version': 1,
+        'formatters': {
+            'request': {
+                'class': f"{__name__}.{ArgusRequestLogFormatter.__name__}",
+                'format': LOG_FORMAT_REQUEST,
+            }
+        },
+        'handlers': {
+            'main': {
+                'class': 'logging.StreamHandler',
+                'stream': 'ext://sys.stderr',
+                'formatter': 'request'
+            }
+        },
+        'loggers': {
+            'cassandra': {
+                'level': log_level,
+                'handlers': ['main']
+            },
+            'argus': {
+                'level': log_level,
+                'handlers': ['main']
+            },
+            'argus_backend': {
+                'level': log_level,
+                'handlers': ['main']
+            },
+            'werkzeug': {
+                'level': log_level,
+                'handlers': ['main']
+            },
+            'uwsgi': {
+                'level': log_level,
+                'handlers': ['main']
+            },
+            '__main__': {
+                'level': log_level,
+                'handlers': ['main']
+            },
+        }
+    })

argus/backend/util/module_loaders.py

@@ -0,0 +1,30 @@
+import sys
+from functools import wraps
+from typing import Callable
+
+
+def is_filter(filter_name: str) -> Callable:
+    def outer_wrapper(func):
+        func.is_filter = True
+        func.filter_name = filter_name
+
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return outer_wrapper
+
+
+def export_functions(module_name: str, attr: str) -> list[Callable]:
+    module = sys.modules[module_name]
+    funcs = []
+
+    for member in dir(module):
+        export = getattr(module, member)
+        applicable_export = getattr(export, attr, False)
+        if applicable_export:
+            funcs.append(export)
+
+    return funcs

argus/backend/util/send_email.py

@@ -0,0 +1,91 @@
+import smtplib
+from typing import List, Set
+from smtplib import SMTPException
+
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from flask import current_app
+from flask import render_template
+
+
+class Email:
+    """
+    Responsible for sending emails
+    """
+    _attachments_size_limit = 10485760  # 10Mb = 20 * 1024 * 1024
+    _body_size_limit = 26214400  # 25Mb = 20 * 1024 * 1024
+
+    def __init__(self, init_connection=True):
+        self.sender: str = ""
+        self._password: str = ""
+        self._user: str = ""
+        self._server_host: str = ""
+        self._server_port: int = 0
+        self._connection: smtplib.SMTP | None = None
+        self._retrieve_credentials()
+        if init_connection:
+            self._connect()
+
+    def _retrieve_credentials(self):
+        self.sender = current_app.config["EMAIL_SENDER"]
+        self._password = current_app.config["EMAIL_SENDER_PASS"]
+        self._user = current_app.config["EMAIL_SENDER_USER"]
+        self._server_host = current_app.config["EMAIL_SERVER"]
+        self._server_port = int(current_app.config["EMAIL_SERVER_PORT"])
+
+    def _connect(self):
+        try:
+            self._connection = smtplib.SMTP(host=self._server_host, port=self._server_port)
+            self._connection.ehlo()
+            self._connection.starttls()
+            self._connection.login(user=self._user, password=self._password)
+        except SMTPException as details:
+            current_app.logger.error("Failed to initialize smtp session %s", details)
+
+    def _is_connection_open(self):
+        if not self._connection:
+            return False
+        try:
+            status, _ = self._connection.noop()
+        except SMTPException:
+            status = -1
+
+        return True if status == 250 else False
+
+    def _prepare_email(self, subject: str,
+                       content: str,
+                       recipients: List[str],
+                       html: bool = True):
+        msg = MIMEMultipart()
+        msg['subject'] = subject
+        msg['from'] = self.sender
+        assert recipients, "No recipients provided"
+        msg['to'] = ','.join(recipients)
+        if html:
+            text_part = MIMEText(content, "html")
+        else:
+            text_part = MIMEText(content, "plain")
+        msg.attach(text_part)
+        email = msg.as_string()
+        return email
+
+    def send(self, subject, content, recipients, html=True):
+        """
+        :param subject: text
+        :param content: text/html
+        :param recipients: iterable, list of recipients
+        :param html: True/False
+        :param files: paths of the files that will be attached to the email
+        :return:
+        """
+        email = self._prepare_email(subject, content, recipients, html)
+        self._send_email(recipients, email)
+
+    def _send_email(self, recipients, email):
+        if not self._is_connection_open():
+            self._connect()
+        self._connection.sendmail(self.sender, recipients, email)
+
+    def __del__(self):
+        if self._connection:
+            self._connection.quit()

argus/client/base.py

@@ -22,7 +22,6 @@ class ArgusClient:
     schema_version: str | None = None
 
     class Routes():
-        # pylint: disable=too-few-public-methods
         SUBMIT = "/testrun/$type/submit"
         GET = "/testrun/$type/$id/get"
         HEARTBEAT = "/testrun/$type/$id/heartbeat"
@@ -144,7 +143,7 @@ class ArgusClient:
         if not (run_type and run_id):
             raise ValueError("run_type and run_id must be set in func params or object attributes")
 
-        response = self.get(endpoint=self.Routes.GET, location_params={"type": run_type, "id": run_id })
+        response = self.get(endpoint=self.Routes.GET, location_params={"type": run_type, "id": run_id})
         self.check_response(response)
 
         return response.json()["response"]
@@ -228,4 +227,3 @@ class ArgusClient:
             }
         )
         self.check_response(response)
-

argus/client/driver_matrix_tests/cli.py

@@ -18,17 +18,22 @@ def cli():
 
 def _submit_driver_result_internal(api_key: str, base_url: str, run_id: str, metadata_path: str, extra_headers: dict):
     metadata = json.loads(Path(metadata_path).read_text(encoding="utf-8"))
-    LOGGER.info("Submitting results for %s [%s/%s] to Argus...", run_id, metadata["driver_name"], metadata["driver_type"])
+    LOGGER.info("Submitting results for %s [%s/%s] to Argus...", run_id,
+                metadata["driver_name"], metadata["driver_type"])
     raw_xml = (Path(metadata_path).parent / metadata["junit_result"]).read_bytes()
     client = ArgusDriverMatrixClient(run_id=run_id, auth_token=api_key, base_url=base_url, extra_headers=extra_headers)
-    client.submit_driver_result(driver_name=metadata["driver_name"], driver_type=metadata["driver_type"], raw_junit_data=base64.encodebytes(raw_xml))
+    client.submit_driver_result(
+        driver_name=metadata["driver_name"], driver_type=metadata["driver_type"], raw_junit_data=base64.encodebytes(raw_xml))
     LOGGER.info("Done.")
 
+
 def _submit_driver_failure_internal(api_key: str, base_url: str, run_id: str, metadata_path: str, extra_headers: dict):
     metadata = json.loads(Path(metadata_path).read_text(encoding="utf-8"))
-    LOGGER.info("Submitting failure for %s [%s/%s] to Argus...", run_id, metadata["driver_name"], metadata["driver_type"])
+    LOGGER.info("Submitting failure for %s [%s/%s] to Argus...", run_id,
+                metadata["driver_name"], metadata["driver_type"])
     client = ArgusDriverMatrixClient(run_id=run_id, auth_token=api_key, base_url=base_url, extra_headers=extra_headers)
-    client.submit_driver_failure(driver_name=metadata["driver_name"], driver_type=metadata["driver_type"], failure_reason=metadata["failure_reason"])
+    client.submit_driver_failure(
+        driver_name=metadata["driver_name"], driver_type=metadata["driver_type"], failure_reason=metadata["failure_reason"])
     LOGGER.info("Done.")
 
 
@@ -53,7 +58,8 @@ def submit_driver_matrix_run(api_key: str, base_url: str, run_id: str, build_id:
 @click.option("--id", "run_id", required=True, help="UUID (v4 or v1) unique to the job")
 @click.option("--metadata-path", required=True, help="Path to the metadata .json file that contains path to junit xml and other required information")
 def submit_driver_result(api_key: str, base_url: str, run_id: str, metadata_path: str, extra_headers: dict):
-    _submit_driver_result_internal(api_key=api_key, base_url=base_url, run_id=run_id, metadata_path=metadata_path, extra_headers=extra_headers)
+    _submit_driver_result_internal(api_key=api_key, base_url=base_url, run_id=run_id,
+                                   metadata_path=metadata_path, extra_headers=extra_headers)
 
 
 @click.command("fail-driver")
@@ -63,7 +69,8 @@ def submit_driver_result(api_key: str, base_url: str, run_id: str, metadata_path
 @click.option("--id", "run_id", required=True, help="UUID (v4 or v1) unique to the job")
 @click.option("--metadata-path", required=True, help="Path to the metadata .json file that contains path to junit xml and other required information")
 def submit_driver_failure(api_key: str, base_url: str, run_id: str, metadata_path: str, extra_headers: dict):
-    _submit_driver_failure_internal(api_key=api_key, base_url=base_url, run_id=run_id, metadata_path=metadata_path, extra_headers=extra_headers)
+    _submit_driver_failure_internal(api_key=api_key, base_url=base_url, run_id=run_id,
+                                    metadata_path=metadata_path, extra_headers=extra_headers)
 
 
 @click.command("submit-or-fail-driver")
@@ -75,9 +82,11 @@ def submit_driver_failure(api_key: str, base_url: str, run_id: str, metadata_pat
 def submit_or_fail_driver(api_key: str, base_url: str, run_id: str, metadata_path: str, extra_headers: dict):
     metadata = json.loads(Path(metadata_path).read_text(encoding="utf-8"))
     if metadata.get("failure_reason"):
-        _submit_driver_failure_internal(api_key=api_key, base_url=base_url, run_id=run_id, metadata_path=metadata_path, extra_headers=extra_headers)
+        _submit_driver_failure_internal(api_key=api_key, base_url=base_url, run_id=run_id,
+                                        metadata_path=metadata_path, extra_headers=extra_headers)
     else:
-        _submit_driver_result_internal(api_key=api_key, base_url=base_url, run_id=run_id, metadata_path=metadata_path, extra_headers=extra_headers)
+        _submit_driver_result_internal(api_key=api_key, base_url=base_url, run_id=run_id,
+                                       metadata_path=metadata_path, extra_headers=extra_headers)
 
 
 @click.command("submit-env")

argus/client/generic/cli.py

@@ -9,6 +9,7 @@ from argus.client.generic.client import ArgusGenericClient
 
 LOGGER = logging.getLogger(__name__)
 
+
 def validate_extra_headers(ctx, param, value):
     if isinstance(value, dict):
         return value
@@ -36,7 +37,8 @@ def cli():
 def submit_run(api_key: str, base_url: str, id: str, build_id: str, build_url: str, started_by: str, scylla_version: str = None, extra_headers: dict | None = None):
     LOGGER.info("Submitting %s (%s) to Argus...", build_id, id)
     client = ArgusGenericClient(auth_token=api_key, base_url=base_url, extra_headers=extra_headers)
-    client.submit_generic_run(build_id=build_id, run_id=id, started_by=started_by, build_url=build_url, scylla_version=scylla_version)
+    client.submit_generic_run(build_id=build_id, run_id=id, started_by=started_by,
+                              build_url=build_url, scylla_version=scylla_version)
     LOGGER.info("Done.")
 
 
@@ -68,7 +70,7 @@ def trigger_jobs(api_key: str, base_url: str, job_info_file: str, version: str,
         LOGGER.error("File not found: %s", job_info_file)
         exit(128)
     payload = json.load(path.open("rt", encoding="utf-8"))
-    client.trigger_jobs({ "release": release, "version": version, "plan_id": plan_id, **payload })
+    client.trigger_jobs({"release": release, "version": version, "plan_id": plan_id, **payload})
 
 
 cli.add_command(submit_run)

argus/client/generic/client.py

@@ -4,6 +4,7 @@ from argus.client.base import ArgusClient
 
 LOGGER = logging.getLogger(__name__)
 
+
 class ArgusGenericClient(ArgusClient):
     test_type = "generic"
     schema_version: None = "v1"

argus/client/generic_result.py

@@ -1,5 +1,6 @@
 from dataclasses import dataclass, field
 from enum import Enum, auto
+from functools import cached_property
 from typing import Union
 
 
@@ -29,13 +30,15 @@ class ColumnMetadata:
     unit: str
     type: ResultType
     higher_is_better: bool = None
+    visible: bool = True  # controls visibility in UI, True by default
 
     def as_dict(self) -> dict:
         return {
             "name": self.name,
             "unit": self.unit,
             "type": str(self.type),
-            "higher_is_better": self.higher_is_better
+            "higher_is_better": self.higher_is_better,
+            "visible": self.visible,
         }
 
 
@@ -52,6 +55,7 @@ class ValidationRule:
             "fixed_limit": self.fixed_limit
         }
 
+
 class ResultTableMeta(type):
     def __new__(cls, name, bases, dct):
         cls_instance = super().__new__(cls, name, bases, dct)
@@ -85,21 +89,39 @@ class Cell:
 
     def as_dict(self) -> dict:
         cell = {"value_text": self.value} if isinstance(self.value, str) else {"value": self.value}
-        cell.update({
-            "column": self.column,
-            "row": self.row,
-            "status": str(self.status)
-        })
+        cell.update({"column": self.column, "row": self.row, "status": str(self.status)})
         return cell
 
 
 @dataclass
-class GenericResultTable(metaclass=ResultTableMeta):
+class GenericResultTable:
     """
     Base class for all Generic Result Tables in Argus. Use it as a base class for your result table.
     """
-    sut_timestamp: int = 0  # automatic timestamp based on SUT version. Works only with SCT and refers to Scylla version.
+
+    name: str = ""
+    description: str = ""
+    columns: list[ColumnMetadata] = field(default_factory=list)
+    # automatic timestamp based on SUT version. Works only with SCT and refers to Scylla version.
+    sut_timestamp: int = 0
+    sut_package_name: str = ""
     results: list[Cell] = field(default_factory=list)
+    validation_rules: dict[str, ValidationRule] = field(default_factory=dict)
+
+    @cached_property
+    def column_types(self):
+        """Return columns types as a dictionary."""
+        return {column.name: column.type for column in self.columns}
+
+    def __post_init__(self):
+        """Validate validation rules."""
+        for col_name, rule in self.validation_rules.items():
+            if col_name not in self.column_types:
+                raise ValueError(f"ValidationRule column {col_name} not found in the table")
+            if self.column_types[col_name] == ResultType.TEXT:
+                raise ValueError(f"Validation rules don't apply to TEXT columns")
+            if not isinstance(rule, ValidationRule):
+                raise ValueError(f"Validation rule for column {col_name} is not of type ValidationRule")
 
     def as_dict(self) -> dict:
         rows = []
@@ -118,7 +140,7 @@ class GenericResultTable(metaclass=ResultTableMeta):
         return {
             "meta": meta_info,
             "sut_timestamp": self.sut_timestamp,
-            "results": [result.as_dict() for result in self.results]
+            "results": [result.as_dict() for result in self.results],
         }
 
     def add_result(self, column: str, row: str, value: Union[int, float, str], status: Status):
@@ -127,3 +149,20 @@ class GenericResultTable(metaclass=ResultTableMeta):
         if isinstance(value, str) and self.column_types[column] != ResultType.TEXT:
             raise ValueError(f"Column {column} is not of type TEXT")
         self.results.append(Cell(column=column, row=row, value=value, status=status))
+
+
+class StaticGenericResultTable(GenericResultTable):
+    """Results class for static results metainformation, defined in Meta class."""
+
+    def __init__(
+        self, name=None, description=None, columns=None, sut_package_name=None, validation_rules=None
+    ):
+        meta = getattr(self.__class__, "Meta")
+        super().__init__(
+            name=name or meta.name,
+            description=description or meta.description,
+            columns=columns or getattr(meta, "Columns", getattr(meta, "columns", None)),
+            sut_package_name=sut_package_name or getattr(meta, "sut_package_name", ""),
+            validation_rules=validation_rules or getattr(
+                meta, "ValidationRules", getattr(meta, "validation_rules", {})),
+        )

argus/client/sct/client.py

@@ -18,7 +18,7 @@ class ArgusSCTClient(ArgusClient):
         CREATE_RESOURCE = "/sct/$id/resource/create"
         TERMINATE_RESOURCE = "/sct/$id/resource/$name/terminate"
         UPDATE_RESOURCE = "/sct/$id/resource/$name/update"
-        SET_SCT_RUNNER  = "/sct/$id/sct_runner/set"
+        SET_SCT_RUNNER = "/sct/$id/sct_runner/set"
         UPDATE_SHARDS_FOR_RESOURCE = "/sct/$id/resource/$name/shards"
         SUBMIT_NEMESIS = "/sct/$id/nemesis/submit"
         SUBMIT_GEMINI_RESULTS = "/sct/$id/gemini/submit"
@@ -209,7 +209,6 @@ class ArgusSCTClient(ArgusClient):
         )
         self.check_response(response)
 
-
     def update_resource(self, name: str, update_data: dict[str, Any]) -> None:
         """
             Update fields of the resource.
@@ -224,7 +223,6 @@ class ArgusSCTClient(ArgusClient):
         )
         self.check_response(response)
 
-
     def submit_nemesis(self, name: str, class_name: str, start_time: int,
                        target_name: str, target_ip: str, target_shards: int) -> None:
         """

argus/client/sirenada/client.py

@@ -12,6 +12,7 @@ from argus.client.base import ArgusClient
 
 LOGGER = logging.getLogger(__name__)
 
+
 class SirenadaEnv(TypedDict):
     SIRENADA_JOB_ID: str
     SIRENADA_BROWSER: str
@@ -31,6 +32,7 @@ class TestCredentials(TypedDict):
     ClusterID: str
     region: str
 
+
 class ArgusSirenadaClient(ArgusClient):
     test_type = "sirenada"
     schema_version: None = "v1"
@@ -138,7 +140,8 @@ class ArgusSirenadaClient(ArgusClient):
             raise exc
 
         credentials = self._read_credentials(self.results_path / self._credentials_filename)
-        results = self._parse_junit_results(junit_xml=self.results_path / self._junit_xml_filename, credentials=credentials, env=env)
+        results = self._parse_junit_results(junit_xml=self.results_path /
+                                            self._junit_xml_filename, credentials=credentials, env=env)
         request_body: RawSirenadaRequest = {}
 
         request_body["run_id"] = env.get("SIRENADA_JOB_ID")

argus/client/tests/conftest.py

@@ -0,0 +1,19 @@
+import shutil
+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture(scope="module", autouse=True)
+def test_dir():
+    return Path(__file__).parent
+
+
+@pytest.fixture(scope="module", autouse=True)
+def env_dir(test_dir):
+    env_dir = test_dir / 'test_env'
+    if env_dir.exists():
+        shutil.rmtree(env_dir)
+    yield env_dir
+    if env_dir.exists():
+        shutil.rmtree(env_dir)

argus/client/tests/test_package.py

@@ -0,0 +1,45 @@
+import subprocess
+import venv
+from pathlib import Path
+
+import pytest
+
+
+def run_command(command: list[str], cwd: str = None, env=None) -> subprocess.CompletedProcess:
+    result = subprocess.run(command, cwd=cwd, check=True, stdout=subprocess.PIPE,
+                            stderr=subprocess.PIPE, text=True, env=env)
+    print(result.stdout)
+    return result
+
+
+@pytest.fixture(scope='module', name='build_and_install')
+def fixture_build_and_install(test_dir: Path, env_dir: Path):
+    """Fixture to build and install the package."""
+    dist_dir = env_dir / 'dist'
+
+    # Build the package
+    run_command(['uv', 'build', '-o', str(dist_dir)], cwd=str(test_dir.parent.parent.parent))
+
+    package_path = next(dist_dir.glob("argus_alm-*-py3-none-any.whl"))
+
+    # install
+    run_command(['uv', 'tool', 'install', str(package_path)])
+
+    yield package_path
+
+    run_command(['uv', 'tool', 'uninstall', 'argus-alm'])
+
+
+def test_should_import_installed_package(env_dir):
+
+    python_exec = ['uv', 'tool', 'run', '--from', 'argus-alm', 'python']
+    run_command(python_exec + ['-c', 'import argus.client; import argus.common; '
+                               'from argus.client.sct.client import ArgusSCTClient'])
+    with pytest.raises(subprocess.CalledProcessError):
+        run_command(python_exec + ['-c', 'import argus.client.tests.test_package'])
+
+
+def test_should_run_cli(build_and_install):
+    """Test that the CLI can be run successfully."""
+    run_command(['uv', 'tool', 'run', '--from', 'argus-alm', 'argus-client-generic', '--help'])
+    run_command(['uv', 'tool', 'run', '--from', 'argus-alm', 'argus-driver-matrix-client', '--help'])