smart-tests-cli 2.0.0__py3-none-any.whl

smart_tests/commands/verify.py

@@ -0,0 +1,153 @@
+import os
+import platform
+import re
+import subprocess
+from typing import List
+
+import click
+
+import smart_tests.args4p.typer as typer
+from smart_tests.utils.commands import Command
+from smart_tests.utils.env_keys import TOKEN_KEY
+from smart_tests.utils.tracking import Tracking, TrackingClient
+
+from .. import args4p
+from ..app import Application
+from ..utils.authentication import get_org_workspace
+from ..utils.java import get_java_command
+from ..utils.smart_tests_client import SmartTestsClient
+from ..utils.typer_types import emoji
+from ..version import __version__ as version
+
+
+def compare_version(a: List[int], b: List[int]):
+    """Compare two version numbers represented as int arrays"""
+
+    def pick(a, i):
+        return a[i] if i < len(a) else 0
+
+    for i in range(max(len(a), len(b))):
+        d = pick(a, i) - pick(b, i)
+        if d != 0:
+            return d  # if they are different, we have the result
+    return 0  # identical
+
+
+def compare_java_version(output: str) -> int:
+    """Check if the Java version meets what we need. returns >=0 if we meet the requirement"""
+    pattern = re.compile('"([^"]+)"')
+    for line in output.splitlines():
+        if line.find("java version") != -1:
+            # line is like: java version "1.8.0_144"
+            m = pattern.search(line)
+            if m:
+                tokens = m.group(1).split(".")
+                if len(tokens) >= 2:
+                    versions = [int(x) for x in tokens[0:2]]
+                    required = [1, 8]
+                    return compare_version(versions, required)
+    # couldn't determine, so err on the safe side
+    return 0
+
+
+def check_java_version(javacmd: str) -> int:
+    """Check if the Java version meets what we need. returns >=0 if we meet the requirement"""
+    try:
+        v = subprocess.run([javacmd, "-version"], check=True, stderr=subprocess.PIPE, universal_newlines=True)
+        return compare_java_version(v.stderr)
+    except subprocess.CalledProcessError:
+        return -1
+
+
+@args4p.command(help="Verify CLI setup and connectivity")
+def verify(app_instance: Application):
+    # Run the verification (no subcommands in this app)
+    # In this command, regardless of REPORT_ERROR_KEY, always report an unexpected error with full stack trace
+    # to assist troubleshooting. `typer.BadParameter` is handled by the invoking
+    # Click gracefully.
+
+    org, workspace = get_org_workspace()
+    tracking_client = TrackingClient(Command.VERIFY, app=app_instance)
+    client = SmartTestsClient(tracking_client=tracking_client, app=app_instance)
+    java = get_java_command()
+
+    # Print the system information first so that we can get them even if there's
+    # an issue.
+
+    click.echo("Organization: " + repr(org))
+    click.echo("Workspace: " + repr(workspace))
+    click.echo("Proxy: " + repr(os.getenv("HTTPS_PROXY")))
+    click.echo("Platform: " + repr(platform.platform()))
+    click.echo("Python version: " + repr(platform.python_version()))
+    click.echo("Java command: " + repr(java))
+    click.echo("smart-tests version: " + repr(version))
+
+    if org is None or workspace is None:
+        msg = (
+            "Could not identify Smart Tests organization/workspace. "
+            "Please confirm if you set SMART_TESTS_TOKEN or SMART_TESTS_ORGANIZATION and SMART_TESTS_WORKSPACE "
+            "environment variables"
+        )
+        tracking_client.send_error_event(
+            event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR,
+            stack_trace=msg
+        )
+        click.secho(msg, fg='red', err=True)
+        raise typer.Exit(1)
+
+    try:
+        res = client.request("get", "verification")
+        if res.status_code == 401:
+            if os.getenv(TOKEN_KEY):
+                msg = ("Authentication failed. Most likely the value for the SMART_TESTS_TOKEN "
+                       "environment variable is invalid.")
+            else:
+                msg = ("Authentication failed. Please set the SMART_TESTS_TOKEN. "
+                       "If you intend to use tokenless authentication, "
+                       "kindly reach out to our support team for further assistance.")
+            click.secho(msg, fg='red', err=True)
+            tracking_client.send_error_event(
+                event_name=Tracking.ErrorEvent.USER_ERROR,
+                stack_trace=msg,
+            )
+            raise typer.Exit(2)
+        res.raise_for_status()
+    except Exception as e:
+        tracking_client.send_error_event(
+            event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR,
+            stack_trace=str(e),
+            api="verification",
+        )
+        client.print_exception_and_recover(e)
+
+    if java is None:
+        msg = "Java is not installed. Install Java version 8 or newer to use the Smart Tests CLI."
+        tracking_client.send_error_event(
+            event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR,
+            stack_trace=msg
+        )
+        click.secho(msg, fg='red', err=True)
+        raise typer.Exit(1)
+
+    # Level 2 check: versions. This is more fragile than just reporting the number, so we move
+    # this out here
+
+    if compare_version([int(x) for x in platform.python_version().split('.')], [3, 6]) < 0:
+        msg = "Python 3.6 or later is required"
+        tracking_client.send_error_event(
+            event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR,
+            stack_trace=msg
+        )
+        click.secho(msg, fg='red', err=True)
+        raise typer.Exit(1)
+
+    if check_java_version(java) < 0:
+        msg = "Java 8 or later is required"
+        tracking_client.send_error_event(
+            event_name=Tracking.ErrorEvent.INTERNAL_CLI_ERROR,
+            stack_trace=msg
+        )
+        click.secho(msg, fg='red', err=True)
+        raise typer.Exit(1)
+
+    click.secho("Your CLI configuration is successfully verified" + emoji(" \U0001f389"), fg='green')

smart_tests/test_runners/adb.py

@@ -0,0 +1,24 @@
+import re
+
+from ..commands.subset import Subset
+from . import smart_tests
+
+
+@smart_tests.subset
+def subset(client: Subset):
+    prev_cls_name = None
+    pattern = re.compile(r'^INSTRUMENTATION_STATUS: class=(.+)$')
+    for line in client.stdin():
+        match = pattern.match(line)
+        if match:
+            cls_name = match.group(1)
+            if prev_cls_name != cls_name:
+                client.test_path([{"type": "class", "name": cls_name}])
+                prev_cls_name = cls_name
+
+    client.separator = ','
+
+    client.run()
+
+
+record_tests = smart_tests.CommonRecordTestImpls(__name__).report_files()

smart_tests/test_runners/ant.py

@@ -0,0 +1,35 @@
+import os
+from typing import Annotated, List
+
+import smart_tests.args4p.typer as typer
+
+from ..commands.subset import Subset
+from ..utils.file_name_pattern import jvm_test_pattern
+from . import smart_tests
+
+
+@smart_tests.subset
+def subset(
+    client: Subset,
+    source_roots: Annotated[List[str], typer.Argument(
+        multiple=True,
+        required=False,
+        help="Source directories to scan for test files"
+    )] = []
+):
+    def file2test(f: str):
+        if jvm_test_pattern.match(f):
+            f = f[:f.rindex('.')]   # remove extension
+            # directory -> package name conversion
+            cls_name = f.replace(os.path.sep, '.')
+            return [{"type": "class", "name": cls_name}]
+        else:
+            return None
+
+    for root in source_roots:
+        client.scan(root.rstrip('/'), "**/*Test.java", file2test)
+
+    client.run()
+
+
+record_tests = smart_tests.CommonRecordTestImpls(__name__).report_files()

smart_tests/test_runners/bazel.py

@@ -0,0 +1,103 @@
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Annotated, Generator, List
+
+from junitparser import TestCase, TestSuite  # type: ignore
+
+import smart_tests.args4p.typer as typer
+
+from ..args4p.converters import path
+from ..commands.subset import Subset
+from ..testpath import TestPath
+from ..utils.logger import Logger
+from . import smart_tests
+
+
+def make_test_path(pkg, target) -> TestPath:
+    return [{'type': 'package', 'name': pkg}, {'type': 'target', 'name': target}]
+
+
+@smart_tests.subset
+def subset(client: Subset):
+    # Read targets from stdin, which generally looks like //foo/bar:zot
+    for label in client.stdin():
+        # //foo/bar:zot -> //foo/bar & zot
+        if label.startswith('//'):
+            pkg, target = label.rstrip('\n').split(':')
+            # TODO: error checks and more robustness
+            client.test_path(make_test_path(pkg.lstrip('//'), target))
+
+    client.formatter = lambda x: x[0]['name'] + ":" + x[1]['name']
+    client.run()
+
+
+smart_tests.CommonDetectFlakesImpls(__name__, formatter=lambda x: x[0]['name'] + ":" + x[1]['name']).detect_flakes()
+
+
+@smart_tests.record.tests
+def record_tests(
+    client,
+    workspace: Annotated[str, typer.Argument(help="Bazel workspace directory")],
+    build_event_json_files: Annotated[List[Path] | None, typer.Option(
+        "--build-event-json",
+        help="set file path generated by --build_event_json_file",
+        multiple=True,
+        type=path(exists=True)
+    )] = None,
+):
+    """
+    Takes Bazel workspace, then report all its test results
+    """
+    base = Path(workspace).joinpath('bazel-testlogs').resolve()
+    if not base.exists():
+        sys.exit("No such directory: %s" % str(base))
+
+    default_path_builder = client.path_builder
+
+    def f(case: TestCase, suite: TestSuite, report_file: str) -> TestPath:
+        # In Bazel, report path name contains package & target.
+        # for example, for //foo/bar:zot, the report file is at bazel-testlogs/foo/bar/zot/test.xml
+        # TODO: robustness
+        pkgNtarget = report_file[len(str(base)) + 1:-len("/test.xml")]
+
+        # last path component is the target, the rest is package
+        # TODO: does this work correctly when on Windows?
+        path = make_test_path(os.path.dirname(pkgNtarget), os.path.basename(pkgNtarget))
+
+        # let the normal path building kicks in
+        path.extend(default_path_builder(case, suite, report_file))
+        return path
+
+    client.path_builder = f
+    client.check_timestamp = False
+
+    if build_event_json_files:
+        for l in parse_build_event_json(build_event_json_files):
+            if l is None:
+                continue
+
+            client.report(str(Path(base).joinpath(l, 'test.xml')))
+    else:
+        client.scan(str(base), '**/test.xml')
+
+    client.run()
+
+
+def parse_build_event_json(files: List[Path]) -> Generator:
+    for file in files:
+        with open(file) as f:
+            for line in f:
+                try:
+                    d = json.loads(line)
+                except Exception:
+                    Logger().error(f"Can not parse build event json {line}")
+                    yield
+                if "id" in d:
+                    if "testResult" in d["id"]:
+                        if "label" in d["id"]["testResult"]:
+                            label = d["id"]["testResult"]["label"]
+                            # replace //foo/bar:zot to /foo/bar/zot
+                            label = label.lstrip("/").replace(":", "/")
+                            yield label

smart_tests/test_runners/behave.py

@@ -0,0 +1,62 @@
+import os
+from typing import Annotated, List, cast
+from xml.etree import ElementTree as ET
+
+import smart_tests.args4p.typer as typer
+
+from ..commands.record.tests import RecordTests
+from ..commands.subset import Subset
+from . import smart_tests
+
+
+@smart_tests.record.tests
+def record_tests(
+    client: RecordTests,
+    reports: Annotated[List[str], typer.Argument(
+        multiple=True,
+        help="Test report files to process"
+    )],
+):
+    for r in reports:
+        client.report(r)
+
+    def parse_func(p: str) -> ET.ElementTree:
+        tree = cast(ET.ElementTree, ET.parse(p))
+        for suite in tree.iter("testsuite"):
+            if len(suite) == 0:
+                continue
+
+            name = suite.get('name')
+            if name is None:
+                continue
+
+            suite_name = name.split('.')
+            if len(suite_name) < 2:
+                continue
+
+            file_name = suite_name[0] + ".feature"
+            class_name = suite_name[1]
+            suite.attrib.update({"filepath": file_name})
+
+            for case in suite:
+                case.attrib.update({"classname": class_name})
+
+        return tree
+
+    client.junitxml_parse_func = parse_func
+    client.run()
+
+
+@smart_tests.subset
+def subset(client: Subset):
+    for t in client.stdin():
+        if 0 < t.find(".feature"):
+            paths = os.path.split(t)
+            if len(paths) < 2:
+                continue
+
+            file = paths[1]
+            client.test_path(file.rstrip('\n'))
+
+    client.separator = "|"
+    client.run()

smart_tests/test_runners/codeceptjs.py

@@ -0,0 +1,33 @@
+import json
+from typing import List
+
+import click
+
+from ..commands.subset import Subset
+from ..testpath import TestPath
+from . import smart_tests
+
+
+@smart_tests.subset
+def subset(client: Subset):
+    def handler(output: List[TestPath], rests: List[TestPath]):
+        # The output would be something like this:
+        # {"tests": ["test/example_test.js", "test/login_test.js"]}
+        if client.rest:
+            with open(client.rest, "w+", encoding="utf-8") as f:
+                f.write(json.dumps({"tests": [client.formatter(t) for t in rests]}))
+        if output:
+            click.echo(json.dumps({"tests": [client.formatter(t) for t in output]}))
+
+    # read lines as test file names
+    for t in client.stdin():
+        if t.rstrip("\n"):
+            client.test_path(t.rstrip("\n"))
+    client.output_handler = handler
+
+    client.run()
+
+
+record_tests = smart_tests.CommonRecordTestImpls(__name__).file_profile_report_files()
+
+# split_subset = smart_tests.CommonSplitSubsetImpls(__name__).split_subset()

smart_tests/test_runners/ctest.py

@@ -0,0 +1,164 @@
+import glob
+import json
+import os
+import re
+from pathlib import Path
+from typing import Annotated, List
+from xml.etree import ElementTree as ET
+
+import click
+
+import smart_tests.args4p.typer as typer
+
+from ..commands.record.tests import RecordTests
+from ..commands.subset import Subset
+from . import smart_tests
+
+
+@smart_tests.subset
+def subset(
+    client: Subset,
+    file: Annotated[str, typer.Argument(
+        help="JSON file to process"
+    )],
+    output_regex_files: Annotated[bool, typer.Option(
+        "--output-regex-files",
+        help="Output test regex to files"
+    )] = False,
+    output_regex_files_dir: Annotated[str, typer.Option(
+        "--output-regex-files-dir",
+        help="Output directory for test regex"
+    )] = ".",
+    output_regex_files_size: Annotated[int, typer.Option(
+        "--output-regex-files-size",
+        help="Max size of each regex file"
+    )] = 60 * 1024,
+):
+    if file:
+        with Path(file).open() as json_file:
+            data = json.load(json_file)
+    else:
+        data = json.loads(client.stdin())   # type: ignore  # TODO
+
+    for test in data['tests']:
+        case = test['name']
+        client.test_path([{'type': 'testcase', 'name': case}])
+
+    if output_regex_files:
+        def handler(output, rests):
+            _write_regex_files(output_regex_files_dir, 'subset', output_regex_files_size, output)
+            _write_regex_files(output_regex_files_dir, 'rest', output_regex_files_size, rests)
+        client.output_handler = handler
+        client.run()
+    else:
+        client.formatter = lambda x: f"^{x[0]['name']}$"
+        client.separator = '|'
+        client.run()
+
+
+def _write_regex_files(output_dir, prefix, max_size, paths):
+    # Python's regexp spec and CTest's regexp spec would be different, but
+    # this escape would work in most of the cases.
+    escaped = _group_by_size(['^' + re.escape(tp[0]['name']) + '$' for tp in paths], max_size)
+    if not os.path.exists(output_dir):
+        os.makedirs(output_dir)
+    for i, elems in enumerate(escaped):
+        with open(os.path.join(output_dir, f"{prefix}_{i}"), 'w') as f:
+            f.write('|'.join(elems) + '\n')
+
+
+def _group_by_size(elems, max_size):
+    ret = []
+    curr = []
+    curr_size = 0
+    for elem in elems:
+        # +1 for the separator
+        if max_size < curr_size + len(elem) + 1:
+            ret.append(curr)
+            curr = [elem]
+            curr_size = len(elem)
+        else:
+            curr.append(elem)
+            curr_size = len(elem) + 1
+    if len(curr) != 0:
+        ret.append(curr)
+    return ret
+
+
+@smart_tests.record.tests
+def record_tests(
+    client: RecordTests,
+    source_roots: Annotated[List[str], typer.Argument(
+        multiple=True,
+        help="Source root directories or files to process"
+    )],
+):
+    for root in source_roots:
+        match = False
+        for t in glob.iglob(root, recursive=True):
+            match = True
+            if os.path.isdir(t):
+                client.scan(t, "*.xml")
+            else:
+                client.report(t)
+        if not match:
+            click.echo(f"No matches found: {root}", err=True)
+
+    def parse_func(p: str) -> ET.ElementTree:
+        """
+        Convert from CTest own XML format to JUnit XML format
+        The projections of these properties are based on
+            https://github.com/rpavlik/jenkins-ctest-plugin/blob/master/ctest-to-junit.xsl
+        """
+        original_tree = ET.parse(p)
+
+        testsuite = ET.Element("testsuite", {"name": "CTest"})
+        test_count = 0
+        failure_count = 0
+        skip_count = 0
+
+        for test in original_tree.findall("./Testing/Test"):
+            test_name = test.find("Name")
+            if test_name is not None:
+                duration_node = test.find("./Results/NamedMeasurement[@name=\"Execution Time\"]/Value")
+                measurement_node = test.find("Results/Measurement/Value")
+
+                stdout = measurement_node.text if measurement_node is not None else ''
+                duration = duration_node.text if duration_node is not None else '0'
+
+                testcase = ET.SubElement(testsuite, "testcase",
+                                         {"name": test_name.text or '',
+                                          "time": str(duration),
+                                          "system-out": stdout or '',
+                                          })
+
+                system_out = ET.SubElement(testcase, "system-out")
+                system_out.text = stdout
+
+                test_count += 1
+                status = test.get("Status")
+                if status is not None:
+                    if status == "failed":
+                        failure = ET.SubElement(testcase, "failure")
+                        failure.text = stdout
+
+                        failure_count += 1
+
+                    if status == "notrun":
+                        skipped = ET.SubElement(testcase, "skipped")
+                        skipped.text = stdout
+
+                        skip_count += 1
+
+        testsuite.attrib.update({
+                                "tests": str(test_count),
+                                "time": "0",
+                                "failures": str(failure_count),
+                                "errors": "0",
+                                "skipped": str(skip_count)
+                                })
+
+        return ET.ElementTree(testsuite)
+
+    client.junitxml_parse_func = parse_func
+    client.run()