smart-tests-cli 2.0.0__py3-none-any.whl

smart_tests/commands/record/tests.py

@@ -0,0 +1,498 @@
+import datetime
+import glob
+import os
+import re
+import xml.etree.ElementTree as ET
+from pathlib import Path
+from time import time_ns
+from typing import Annotated, Callable, Dict, Generator, List, Tuple, Union
+
+import click
+from dateutil.parser import ParserError, parse
+from junitparser import JUnitXml, TestCase, TestSuite  # type: ignore  # noqa: F401
+from more_itertools import ichunked
+from tabulate import tabulate
+
+import smart_tests.args4p.converters as converters
+import smart_tests.args4p.typer as typer
+from smart_tests.utils.authentication import ensure_org_workspace
+from smart_tests.utils.env_keys import REPORT_ERROR_KEY
+from smart_tests.utils.session import get_session, parse_session
+from smart_tests.utils.tracking import Tracking, TrackingClient
+
+from ...app import Application
+from ...args4p.command import Group
+from ...args4p.exceptions import BadCmdLineException
+from ...testpath import FilePathNormalizer, TestPathComponent, unparse_test_path
+from ...utils.commands import Command
+from ...utils.exceptions import InvalidJUnitXMLException, print_error_and_die
+from ...utils.fail_fast_mode import (FailFastModeValidateParams, fail_fast_mode_validate,
+                                     set_fail_fast_mode, warn_and_exit_if_fail_fast_mode)
+from ...utils.logger import Logger
+from ...utils.smart_tests_client import SmartTestsClient
+from .case_event import CaseEvent, CaseEventGenerator, CaseEventType, DataBuilder, TestPathBuilder
+
+GROUP_NAME_RULE = re.compile("^[a-zA-Z0-9][a-zA-Z0-9_-]*$")
+RESERVED_GROUP_NAMES = ["group", "groups", "nogroup", "nogroups"]
+
+
+def _validate_group(value):
+    if value is None:
+        return ""
+
+    if str(value).lower() in RESERVED_GROUP_NAMES:
+        raise BadCmdLineException(f"{value} is reserved name.")
+
+    if GROUP_NAME_RULE.match(value):
+        return value
+    else:
+        raise BadCmdLineException("group option supports only alphabet(a-z, A-Z), number(0-9), '-', and '_'")
+
+
+ParseFunc = Callable[[str], CaseEventGenerator]
+
+
+class RecordTests:
+    # The most generic form of parsing, where a path to a test report
+    # is turned into a generator by using CaseEvent.create()
+
+    # A common mechanism to build ParseFunc by building JUnit XML report in-memory (or build it the usual way
+    # and patch it to fix things up). This is handy as some libraries
+    # produce invalid / broken JUnit reports
+    JUnitXmlParseFunc = Callable[[str], ET.Element | ET.ElementTree]
+
+    path_builder: TestPathBuilder
+
+    parse_func: ParseFunc
+
+    build_name: str
+
+    test_session_id: int
+
+    # session is generated by `smart-tests record session` command
+    # the session format is `builds/<BUILD_NUMBER>/test_sessions/<TEST_SESSION_ID>`
+    session: str
+
+    # This function, if supplied, is used to build a test path
+    # that uniquely identifies a test case
+    metadata_builder: DataBuilder
+
+    # setter only property that sits on top of the parse_func property
+    def set_junitxml_parse_func(self, f: JUnitXmlParseFunc):
+        """
+        Parse XML report file with the JUnit report file, possibly with the custom parser function 'f'
+        that can be used to build JUnit ET.Element tree from scratch or do some patch up.
+
+        If f=None, the default parse code from JUnitParser module is used.
+        """
+
+        def parse(report: str) -> Generator[CaseEventType, None, None]:
+            # To understand JUnit XML format, https://llg.cubic.org/docs/junit/ is helpful
+            # TODO: robustness: what's the best way to deal with broken XML
+            # file, if any?
+            try:
+                xml = JUnitXml.fromfile(report, f)
+            except Exception as e:
+                # `JUnitXml.fromfile()` will raise `JUnitXmlError` and other lxml related errors
+                # if the file has wrong format.
+                # https://github.com/weiwei/junitparser/blob/master/junitparser/junitparser.py#L321
+                warn_and_exit_if_fail_fast_mode(
+                    "Warning: error reading JUnitXml file {filename}: {error}".format(
+                        filename=report, error=e))
+                return
+            if isinstance(xml, JUnitXml):
+                testsuites = [suite for suite in xml]
+            elif isinstance(xml, TestSuite):
+                testsuites = [xml]
+            else:
+                raise InvalidJUnitXMLException(filename=report)
+
+            try:
+                for suite in testsuites:
+                    for case in suite:
+                        yield CaseEvent.from_case_and_suite(self.path_builder, case, suite, report, self.metadata_builder)
+            except Exception as e:
+                warn_and_exit_if_fail_fast_mode(
+                    "Warning: error parsing JUnitXml file {filename}: {error}".format(
+                        filename=report, error=e))
+
+        self.parse_func = parse
+
+    junitxml_parse_func = property(None, set_junitxml_parse_func)
+
+    check_timestamp: bool
+
+    def __init__(
+            self,
+            app: Application,
+            session: Annotated[str, typer.Option(
+                "--session",
+                help="In the format builds/<build-name>/test_sessions/<test-session-id>",
+                required=True
+            )],
+            base_path: Annotated[Path | None, typer.Option(
+                "--base",
+                help="(Advanced) base directory to make test names portable",
+                type=converters.path(exists=True, file_okay=False, dir_okay=True, resolve_path=True),
+            )] = None,
+            post_chunk: Annotated[int, typer.Option(
+                "--post-chunk",
+                help="Post chunk"
+            )] = 1000,
+            no_base_path_inference: Annotated[bool, typer.Option(
+                "--no-base-path-inference",
+                help="Do not guess the base path to relativize the test file paths. By default, if the test file paths are "
+                     "absolute file paths, it automatically guesses the repository root directory and relativize the paths. "
+                     "With this option, the command doesn't do this guess work. If --base-path is specified, the absolute "
+                     "file paths are relativized to the specified path irrelevant to this option. Use it if the guessed base "
+                     "path is incorrect."
+            )] = False,
+            report_paths: Annotated[bool, typer.Option(
+                "--report-paths",
+                help="Instead of POSTing test results, just report test paths in the report file then quit. For diagnostics. "
+                     "Use with --dry-run",
+                hidden=True
+            )] = False,
+            group: Annotated[str | None, typer.Option(
+                help="Grouping name for test results"
+            )] = "",
+            is_allow_test_before_build: Annotated[bool, typer.Option(
+                "--allow-test-before-build",
+                help="",
+                hidden=True
+            )] = False,
+            test_runner: Annotated[str | None, typer.Argument()] = None,
+            # TODO(Konboi): restore timestamp option
+    ):
+        self.logger = Logger()
+
+        self.org, self.workspace = ensure_org_workspace()
+
+        app.test_runner = test_runner
+        self.app = app
+
+        self.tracking_client = TrackingClient(Command.RECORD_TESTS, app=app)
+        self.client = SmartTestsClient(app=app, tracking_client=self.tracking_client)
+        set_fail_fast_mode(self.client.is_fail_fast_mode())
+
+        fail_fast_mode_validate(FailFastModeValidateParams(
+            command=Command.RECORD_TESTS,
+            session=session,
+        ))
+
+        self.post_chunk = post_chunk
+        self.report_paths = report_paths
+
+        # Validate group if provided and ensure it's never None
+        if group is None:
+            group = ""
+        elif group:
+            group = _validate_group(group)
+        self.group = group
+
+        self.file_path_normalizer = FilePathNormalizer(
+            str(base_path) if base_path else None,
+            no_base_path_inference=no_base_path_inference)
+
+        try:
+            test_session = get_session(session, self.client)
+            self.record_start_at = get_record_start_at(session, self.client)
+
+            test_session_id = test_session.id
+            build_name = test_session.build_name
+        except ValueError as e:
+            print_error_and_die(msg=str(e), event=Tracking.ErrorEvent.USER_ERROR, tracking_client=self.tracking_client)
+        except Exception as e:
+            if os.getenv(REPORT_ERROR_KEY):
+                raise e
+
+            self.tracking_client.send_error_event(
+                event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR,
+                stack_trace=str(e),
+            )
+            self.client.print_exception_and_recover(e)
+            # To prevent users from stopping the CI pipeline, the cli exits with a
+            # status code of 0, indicating that the program terminated successfully.
+            build_name, test_session_id = parse_session(session)
+            exit(0)
+
+        self.reports: List[str] = []
+        self.skipped_reports: List[str] = []
+        self.path_builder = CaseEvent.default_path_builder(self.file_path_normalizer)
+        self.junitxml_parse_func = None
+        self.check_timestamp = True
+        self.base_path = str(base_path) if base_path else None
+        self.dry_run = app.dry_run  # TODO: remove
+        self.no_base_path_inference = no_base_path_inference
+        self.is_allow_test_before_build = is_allow_test_before_build
+        self.build_name = build_name
+        self.test_session_id = test_session_id
+        self.session = session
+        self.metadata_builder = CaseEvent.default_data_builder()
+
+    def make_file_path_component(self, filepath) -> TestPathComponent:
+        """Create a single TestPathComponent from the given file path"""
+        if self.base_path:
+            filepath = os.path.relpath(filepath, start=self.base_path)
+        return {"type": "file", "name": filepath}
+
+    def report(self, junit_report_file: str):
+        ctime = datetime.datetime.fromtimestamp(
+            os.path.getctime(junit_report_file))
+
+        if (
+                not self.is_allow_test_before_build  # nlqa: W503
+                and self.check_timestamp  # noqa: W503
+                and ctime.timestamp() < self.record_start_at.timestamp()  # noqa: W503
+        ):
+            format = "%Y-%m-%d %H:%M:%S"
+            self.logger.warning(
+                f"skip: {junit_report_file} is too old to report. start_record_at:"
+                f"{self.record_start_at.strftime(format)} file_created_at: {ctime.strftime(format)}")
+            self.skipped_reports.append(junit_report_file)
+
+            return
+
+        self.reports.append(junit_report_file)
+
+    def scan(self, base: str, pattern: str):
+        """
+        Starting at the 'base' path, recursively add everything that matches the given GLOB pattern
+
+        scan('build/test-reports', '**/*.xml')
+        """
+        for t in glob.iglob(os.path.join(base, pattern), recursive=True):
+            self.report(t)
+
+    def run(self):
+        count = 0  # count number of test cases sent
+        is_observation = False
+
+        def testcases(reports: List[str]) -> Generator[CaseEventType, None, None]:
+            exceptions = []
+            for report in reports:
+                try:
+                    for tc in self.parse_func(report):
+                        # trim empty test path
+                        if len(tc.get('testPath', [])) == 0:
+                            continue
+
+                        # Timestamp option has been removed
+
+                        yield tc
+
+                except Exception as e:
+                    exceptions.append(Exception(f"Failed to process a report file: {report}", e))
+
+            if len(exceptions) > 0:
+                # defer XML parsing exceptions so that we can send what we
+                # can send before we bail out
+                raise Exception(exceptions)
+
+        # generator that creates the payload incrementally
+        def payload(
+                cases: Generator[TestCase, None, None],
+                test_runner, group: str,
+                test_suite_name: str,
+                flavors: Dict[str, str]) -> Tuple[Dict[str, Union[str, List, dict, bool]], List[Exception]]:
+            nonlocal count
+            cs = []
+            exs = []
+
+            while True:
+                try:
+                    cs.append(next(cases))
+                except StopIteration:
+                    break
+                except Exception as ex:
+                    exs.append(ex)
+
+            count += len(cs)
+            return {
+                "events": cs,
+                "testRunner": test_runner,
+                "group": group,
+                "metadata": get_env_values(self.client),
+                "noBuild": False,  # deprecated to set no-build from the record tests command
+                # NOTE:
+                # testSuite and flavors are applied only when the no-build option is enabled
+                "testSuite": test_suite_name,
+                "flavors": flavors,
+            }, exs
+
+        def send(payload: Dict[str, Union[str, List]]) -> None:
+            res = self.client.request(
+                "post", f"{self.session}/events", payload=payload, compress=True)
+            res.raise_for_status()
+
+            nonlocal is_observation
+            is_observation = res.json().get("testSession", {}).get("isObservation", False)
+
+        def recorded_result() -> Tuple[int, int, int, float]:
+            test_count = 0
+            success_count = 0
+            fail_count = 0
+            duration = float(0)
+
+            for tc in testcases(self.reports):
+                test_count += 1
+                status = tc.get("status")
+                if status == 0:
+                    fail_count += 1
+                elif status == 1:
+                    success_count += 1
+                duration += float(tc.get("duration") or 0)  # sec
+
+            return test_count, success_count, fail_count, duration / 60   # sec to min
+
+        try:
+            start = time_ns()
+            tc = testcases(self.reports)
+            end = time_ns()
+            self.tracking_client.send_event(
+                event_name=Tracking.Event.PERFORMANCE,
+                metadata={
+                    "elapsedTime": end - start,
+                    "measurementTarget": "testcases method(parsing report file)"
+                }
+            )
+
+            if self.report_paths:
+                # diagnostics mode to just report test paths
+                for t in tc:
+                    print(unparse_test_path(t['testPath']))
+                return
+
+            start = time_ns()
+            exceptions = []
+            for chunk in ichunked(tc, self.post_chunk):
+                p, es = payload(
+                    cases=chunk,
+                    test_runner=self.app.test_runner,
+                    group=self.group,
+                    test_suite_name="",  # test_suite option was removed
+                    flavors={},  # flavor option was removed
+                )
+
+                send(p)
+                exceptions.extend(es)
+            end = time_ns()
+            self.tracking_client.send_event(
+                event_name=Tracking.Event.PERFORMANCE,
+                metadata={
+                    "elapsedTime": end - start,
+                    "measurementTarget": "events API"
+                }
+            )
+
+            if len(exceptions) > 0:
+                raise Exception(exceptions)
+
+        except Exception as e:
+            self.tracking_client.send_error_event(
+                event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR,
+                stack_trace=str(e),
+            )
+            self.client.print_exception_and_recover(e)
+            return
+
+        if count == 0:
+            if len(self.skipped_reports) != 0:
+                warn_and_exit_if_fail_fast_mode(
+                    "{} test report(s) were skipped because they were created before this build was recorded.\n"
+                    "Make sure to run your tests after you run `smart-tests record build`.\n"
+                    "Otherwise, if these are really correct test reports, use the `--allow-test-before-build` option.".
+                    format(len(self.skipped_reports)))
+                return
+            else:
+                warn_and_exit_if_fail_fast_mode(
+                    "Looks like tests didn't run? If not, make sure the right files/directories were passed into `smart-tests record tests`")  # noqa: E501
+                return
+
+        file_count = len(self.reports)
+        test_count, success_count, fail_count, duration = recorded_result()
+
+        click.echo(
+            f"Smart Tests recorded tests for build "
+            f"{self.build_name}(test session {self.test_session_id}) to "
+            f"workspace {self.org} / {self.workspace} from {file_count} files: ")
+
+        if is_observation:
+            click.echo("(This test session is under observation mode)")
+
+        click.echo("")
+
+        header = ["Files found", "Tests found", "Tests passed", "Tests failed", "Total duration (min)"]
+
+        rows = [[file_count, test_count, success_count, fail_count, duration]]
+        click.echo(tabulate(rows, header, tablefmt="github", floatfmt=".2f"))
+
+        if duration == 0:
+            click.secho("\nTotal test duration is 0."
+                        "\nPlease check whether the test duration times in report files are correct.", fg="yellow")
+        click.echo(
+            f"\nVisit https://app.launchableinc.com/organizations/{self.org}/workspaces/"
+            f"{self.workspace}/test-sessions/{self.test_session_id} to view uploaded test results "
+            f"(or run `launchable inspect tests --test-session-id {self.test_session_id}`)")
+
+
+tests = Group(name="tests", callback=RecordTests, help="Record test results")
+
+
+# if we fail to determine the timestamp of the build, we err on the side of collecting more test reports
+# than no test reports, so we use the 'epoch' timestamp
+INVALID_TIMESTAMP = datetime.datetime.fromtimestamp(0)
+
+
+def get_record_start_at(session: str, client: SmartTestsClient):
+    """
+    Determine the baseline timestamp to be used for up-to-date checks of report files.
+    Only files newer than this timestamp will be collected.
+
+    Based on the thinking that if a build doesn't exist tests couldn't have possibly run, we attempt
+    to use the timestamp of a build, with appropriate fallback.
+    """
+    build_name, _ = parse_session(session)
+
+    sub_path = f"builds/{build_name}"
+
+    res = client.request("get", sub_path)
+    if res.status_code != 200:
+        if res.status_code == 404:
+            msg = "Build {} was not found. " \
+                  f"Make sure to run `smart-tests record build --name {build_name}` before `smart-tests record tests`"
+        else:
+            msg = f"Unable to determine the timestamp of the build {build_name}. HTTP response code was {res.status_code}"
+        click.secho(msg, fg='yellow', err=True)
+
+        # to avoid stop report command
+        return INVALID_TIMESTAMP
+
+    created_at = res.json()["createdAt"]
+    Logger().debug(f"Build {build_name} timestamp = {created_at}")
+    t = parse_launchable_timeformat(created_at)
+    return t
+
+
+def parse_launchable_timeformat(t: str) -> datetime.datetime:
+    # e.g) "2021-04-01T09:35:47.934+00:00"
+    try:
+        return parse(t)
+    except ParserError:
+        return INVALID_TIMESTAMP
+
+
+def get_env_values(client: SmartTestsClient) -> Dict[str, str]:
+    sub_path = "slack/notification/key/list"
+    res = client.request("get", sub_path=sub_path)
+
+    metadata: Dict[str, str] = {}
+    if res.status_code != 200:
+        return metadata
+
+    keys = res.json().get("keys", [])
+    for key in keys:
+        val = os.getenv(key, "")
+        metadata[key] = val
+
+    return metadata

smart_tests/commands/stats/__init__.py

@@ -0,0 +1,11 @@
+from ... import args4p
+from ...app import Application
+from .test_sessions import test_sessions
+
+
+@args4p.group()
+def stats(app: Application):
+    return app
+
+
+stats.add_command(test_sessions)

smart_tests/commands/stats/test_sessions.py

@@ -0,0 +1,45 @@
+from typing import Annotated, Any, Dict, List
+
+import click
+
+import smart_tests.args4p.typer as typer
+
+from ... import args4p
+from ...app import Application
+from ...utils.smart_tests_client import SmartTestsClient
+from ...utils.typer_types import validate_key_value
+
+
+@args4p.command(help="View test session statistics")
+def test_sessions(
+    app: Application,
+    days: Annotated[int, typer.Option(
+        help="How many days of test sessions in the past to be stat"
+    )] = 7,
+    flavor: Annotated[List[str], typer.Option(
+        multiple=True,
+        help="flavors",
+        metavar="KEY=VALUE"
+    )] = [],
+):
+    # Parse flavors
+    parsed_flavors = [validate_key_value(f) for f in flavor]
+
+    params: Dict[str, Any] = {'days': days, 'flavor': []}
+    flavors = []
+    for f in parsed_flavors:
+        flavors.append('%s=%s' % (f[0], f[1]))
+
+    if flavors:
+        params['flavor'] = flavors
+    else:
+        params.pop('flavor', None)
+
+    client = SmartTestsClient(app=app)
+    try:
+        res = client.request('get', '/stats/test-sessions', params=params)
+        res.raise_for_status()
+        click.echo(res.text)
+
+    except Exception as e:
+        client.print_exception_and_recover(e, "Warning: the service failed to get stat.")