tft-cli 0.0.17__tar.gz → 0.0.18__tar.gz

{tft_cli-0.0.17 → tft_cli-0.0.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tft-cli
-Version: 0.0.17
+Version: 0.0.18
 Summary: Testing Farm CLI tool
 License: Apache-2.0
 Author: Miroslav Vadkerti
@@ -15,4 +15,5 @@ Requires-Dist: click (>=8.0.4,<8.1.0)
 Requires-Dist: colorama (>=0.4.4,<0.5.0)
 Requires-Dist: dynaconf (>=3.1.7,<4.0.0)
 Requires-Dist: requests (>=2.27.1,<3.0.0)
+Requires-Dist: ruamel-yaml (>=0.18.6,<0.19.0)
 Requires-Dist: typer[all] (>=0.7.0,<0.8.0)

{tft_cli-0.0.17 → tft_cli-0.0.18}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tft-cli"
-version = "0.0.17"
+version = "0.0.18"
 description = "Testing Farm CLI tool"
 authors = ["Miroslav Vadkerti <mvadkert@redhat.com>"]
 license = "Apache-2.0"
@@ -21,6 +21,7 @@ click = "~8.0.4"
 dynaconf = "^3.1.7"
 colorama = "^0.4.4"
 requests = "^2.27.1"
+ruamel-yaml = "^0.18.6"
 
 [tool.poetry.dev-dependencies]
 pyre-check = "^0.9.10"

{tft_cli-0.0.17 → tft_cli-0.0.18}/src/tft/cli/commands.py

@@ -10,6 +10,7 @@ import subprocess
 import textwrap
 import time
 import urllib.parse
+import xml.etree.ElementTree as ET
 from enum import Enum
 from typing import Any, Dict, List, Optional
 
@@ -18,12 +19,14 @@ import requests
 import typer
 from rich import print
 from rich.progress import Progress, SpinnerColumn, TextColumn
+from rich.table import Table
 
 from tft.cli.config import settings
 from tft.cli.utils import (
     artifacts,
     cmd_output_or_exit,
     console,
+    console_stderr,
     exit_error,
     hw_constraints,
     install_http_retries,
@@ -57,6 +60,11 @@ RESERVE_REF = os.getenv("TESTING_FARM_RESERVE_REF", "main")
 DEFAULT_PIPELINE_TIMEOUT = 60 * 12
 
 
+class WatchFormat(str, Enum):
+    text = 'text'
+    json = 'json'
+
+
 class PipelineType(str, Enum):
     tmt_multihost = "tmt-multihost"
 
@@ -127,11 +135,11 @@ OPTION_POST_INSTALL_SCRIPT: Optional[str] = typer.Option(
 )
 OPTION_KICKSTART: Optional[List[str]] = typer.Option(
     None,
-    metavar="key=value",
+    metavar="key=value|@file",
     help=(
         "Kickstart specification to customize the guest installation. Expressed as a key=value pair. "
         "For more information about the supported keys see "
-        "https://tmt.readthedocs.io/en/stable/spec/plans.html#kickstart."
+        "https://tmt.readthedocs.io/en/stable/spec/plans.html#kickstart. The @ prefix marks a yaml file to load."
     ),
 )
 OPTION_POOL: Optional[str] = typer.Option(
@@ -169,19 +177,27 @@ OPTION_DRY_RUN: bool = typer.Option(
     False, help="Do not submit a request to Testing Farm, just print it.", rich_help_panel=RESERVE_PANEL_GENERAL
 )
 OPTION_VARIABLES: Optional[List[str]] = typer.Option(
-    None, "-e", "--environment", metavar="key=value", help="Variables to pass to the test environment."
+    None,
+    "-e",
+    "--environment",
+    metavar="key=value|@file",
+    help="Variables to pass to the test environment. The @ prefix marks a yaml file to load.",
 )
 OPTION_SECRETS: Optional[List[str]] = typer.Option(
-    None, "-s", "--secret", metavar="key=value", help="Secret variables to pass to the test environment."
+    None,
+    "-s",
+    "--secret",
+    metavar="key=value|@file",
+    help="Secret variables to pass to the test environment. The @ prefix marks a yaml file to load.",
 )
 OPTION_HARDWARE: List[str] = typer.Option(
     None,
     help=(
         "HW requirements, expressed as key/value pairs. Keys can consist of several properties, "
-        "e.g. ``disk.space='>= 40 GiB'``, such keys will be merged in the resulting environment "
-        "with other keys sharing the path: ``cpu.family=79`` and ``cpu.model=6`` would be merged, "
-        "not overwriting each other. See https://tmt.readthedocs.io/en/stable/spec/hardware.html "
-        "for the hardware specification."
+        "e.g. ``disk.size='>= 40 GiB'``, such keys will be merged in the resulting environment "
+        "with other keys sharing the path: ``cpu.family=79`` and ``cpu.model=6`` would be merged, not overwriting "
+        "each other. See https://docs.testing-farm.io/Testing%20Farm/0.1/test-request.html#hardware "
+        "for the supported hardware selection possibilities."
     ),
 )
 OPTION_WORKER_IMAGE: Optional[str] = typer.Option(
@@ -197,11 +213,157 @@ OPTION_PARALLEL_LIMIT: Optional[int] = typer.Option(
 )
 
 
+def _parse_xunit(xunit: str):
+    """
+    A helper that parses xunit file into sets of passed_plans/failed_plans/errored_plans per arch.
+
+    The plans are returned as a {'arch': ['plan1', 'plan2', ..]} map. If it was impossible to deduce architecture
+    from a certain plan result (happens in case of early fails / infra issues), the plan will be listed under the 'N/A'
+    key.
+    """
+
+    def _add_plan(collection: dict, arch: str, plan: ET.Element):
+        # NOTE(ivasilev) name property will always be defined at this point, defaulting to '' to make type check happy
+        plan_name = plan.get('name', '')
+        if arch in collection:
+            collection[arch].append(plan_name)
+        else:
+            collection[arch] = [plan_name]
+
+    failed_plans = {}
+    passed_plans = {}
+    errored_plans = {}
+
+    results_root = ET.fromstring(xunit)
+    for plan in results_root.findall('./testsuite'):
+        # Try to get information about the environment (stored under testcase/testing-environment), may be
+        # absent if state is undefined
+        testing_environment: Optional[ET.Element] = plan.find('./testcase/testing-environment[@name="requested"]')
+        if not testing_environment:
+            console_stderr.print(
+                f'Could not find env specifications for {plan.get("name")}, assuming fail for all arches'
+            )
+            arch = 'N/A'
+        else:
+            arch_property = testing_environment.find('./property[@name="arch"]')
+            if arch_property is None:
+                console_stderr.print(f'Could not find arch property for plan {plan.get("name")} results, skipping')
+                continue
+            # NOTE(ivasilev) arch property will always be defined at this point, defaulting to '' to make type check
+            # happy
+            arch = arch_property.get('value', '')
+        if plan.get('result') == 'passed':
+            _add_plan(passed_plans, arch, plan)
+        elif plan.get('result') == 'failed':
+            _add_plan(failed_plans, arch, plan)
+        else:
+            _add_plan(errored_plans, arch, plan)
+
+    # Let's remove possible duplicates among N/A errored out tests
+    if 'N/A' in errored_plans:
+        errored_plans['N/A'] = list(set(errored_plans['N/A']))
+    return passed_plans, failed_plans, errored_plans
+
+
+def _get_request_summary(request: dict, session: requests.Session):
+    """A helper that prepares json summary of the test run"""
+    state = request.get('state')
+    artifacts_url = (request.get('run') or {}).get('artifacts')
+    xpath_url = f'{artifacts_url}/results.xml' if artifacts_url else ''
+    xunit = (request.get('result') or {}).get('xunit') or '<testsuites></testsuites>'
+    if state not in ['queued', 'running'] and artifacts_url:
+        # NOTE(ivasilev) xunit can be None (ex. in case of timed out requests) so let's fetch results.xml and use it
+        # as source of truth
+        try:
+            response = session.get(xpath_url)
+            if response.status_code == 200:
+                xunit = response.text
+        except requests.exceptions.ConnectionError:
+            console_stderr.print("Could not get xunit results")
+    passed_plans, failed_plans, errored_plans = _parse_xunit(xunit)
+    overall = (request.get("result") or {}).get("overall")
+    arches_requested = [env['arch'] for env in request['environments_requested']]
+
+    return {
+        'id': request['id'],
+        'state': request['state'],
+        'artifacts': artifacts_url,
+        'overall': overall,
+        'arches_requested': arches_requested,
+        'errored_plans': errored_plans,
+        'failed_plans': failed_plans,
+        'passed_plans': passed_plans,
+    }
+
+
+def _print_summary_table(summary: dict, format: Optional[WatchFormat], show_details=True):
+    if not format == WatchFormat.text:
+        # Nothing to do, table is printed only when text output is requested
+        return
+
+    def _get_plans_list(collection):
+        return list(collection.values())[0] if collection.values() else []
+
+    def _has_plan(collection, arch, plan):
+        return plan in collection.get(arch, [])
+
+    # Let's transform plans maps into collection of plans to display plan result per arch statistics
+    errored = _get_plans_list(summary['errored_plans'])
+    failed = _get_plans_list(summary['failed_plans'])
+    passed = _get_plans_list(summary['passed_plans'])
+    generic_info_table = Table(show_header=True, header_style="bold magenta")
+    arches_requested = summary['arches_requested']
+    artifacts_url = summary['artifacts'] or ''
+    for column in summary.keys():
+        generic_info_table.add_column(column)
+    generic_info_table.add_row(
+        summary['id'],
+        summary['state'],
+        f'[link]{artifacts_url}[/link]',
+        summary['overall'],
+        ','.join(arches_requested),
+        str(len(errored)),
+        str(len(failed)),
+        str(len(passed)),
+    )
+    console.print(generic_info_table)
+
+    all_plans = sorted(set(errored + failed + passed))
+    details_table = Table(show_header=True, header_style="bold magenta")
+    for column in ["plan"] + arches_requested:
+        details_table.add_column(column)
+
+    for plan in all_plans:
+        row = [plan]
+        for arch in arches_requested:
+            if _has_plan(summary['passed_plans'], arch, plan):
+                res = '[green]pass[/green]'
+            elif _has_plan(summary['failed_plans'], arch, plan):
+                res = '[red]fail[/red]'
+            elif _has_plan(summary['errored_plans'], 'N/A', plan):
+                res = '[yellow]error[/yellow]'
+            else:
+                # If for some reason the plan has not been executed for this arch (this can happen after
+                # applying adjust rules) -> don't show anything
+                res = None
+            row.append(res)
+        details_table.add_row(*row)
+    if show_details:
+        console.print(details_table)
+
+
 def watch(
     api_url: str = typer.Option(settings.API_URL, help="Testing Farm API URL."),
     id: str = typer.Option(..., help="Request ID to watch"),
     no_wait: bool = typer.Option(False, help="Skip waiting for request completion."),
+    format: Optional[WatchFormat] = typer.Option(WatchFormat.text, help="Output format"),
 ):
+    def _console_print(*args, **kwargs):
+        """A helper function that will skip printing to console if output format is json"""
+        if format == WatchFormat.json:
+            return
+        console.print(*args, **kwargs)
+
     """Watch request for completion."""
 
     if not uuid_valid(id):
@@ -210,10 +372,10 @@ def watch(
     get_url = urllib.parse.urljoin(api_url, f"/v0.1/requests/{id}")
     current_state: str = ""
 
-    console.print(f"🔎 api [blue]{get_url}[/blue]")
+    _console_print(f"🔎 api [blue]{get_url}[/blue]")
 
     if not no_wait:
-        console.print("💡 waiting for request to finish, use ctrl+c to skip", style="bright_yellow")
+        _console_print("💡 waiting for request to finish, use ctrl+c to skip", style="bright_yellow")
 
     artifacts_shown = False
 
@@ -245,37 +407,45 @@ def watch(
 
         current_state = state
 
+        request_summary = _get_request_summary(request, session)
+        if format == WatchFormat.json:
+            console.print(json.dumps(request_summary, indent=2))
+
         if state == "new":
-            console.print("👶 request is [blue]waiting to be queued[/blue]")
+            _console_print("👶 request is [blue]waiting to be queued[/blue]")
 
         elif state == "queued":
-            console.print("👷 request is [blue]queued[/blue]")
+            _console_print("👷 request is [blue]queued[/blue]")
 
         elif state == "running":
-            console.print("🚀 request is [blue]running[/blue]")
-            console.print(f"🚢 artifacts [blue]{request['run']['artifacts']}[/blue]")
+            _console_print("🚀 request is [blue]running[/blue]")
+            _console_print(f"🚢 artifacts [blue]{request['run']['artifacts']}[/blue]")
             artifacts_shown = True
 
         elif state == "complete":
             if not artifacts_shown:
-                console.print(f"🚢 artifacts [blue]{request['run']['artifacts']}[/blue]")
+                _console_print(f"🚢 artifacts [blue]{request['run']['artifacts']}[/blue]")
 
             overall = request["result"]["overall"]
             if overall in ["passed", "skipped"]:
-                console.print("✅ tests passed", style="green")
+                _console_print("✅ tests passed", style="green")
+                _print_summary_table(request_summary, format)
                 raise typer.Exit()
 
             if overall in ["failed", "error", "unknown"]:
-                console.print(f"❌ tests {overall}", style="red")
+                _console_print(f"❌ tests {overall}", style="red")
                 if overall == "error":
-                    console.print(f"{request['result']['summary']}", style="red")
+                    _console_print(f"{request['result']['summary']}", style="red")
+                _print_summary_table(request_summary, format)
                 raise typer.Exit(code=1)
 
         elif state == "error":
-            console.print(f"📛 pipeline error\n{request['result']['summary']}", style="red")
+            _console_print(f"📛 pipeline error\n{request['result']['summary']}", style="red")
+            _print_summary_table(request_summary, format)
             raise typer.Exit(code=2)
 
         if no_wait:
+            _print_summary_table(request_summary, format, show_details=False)
             raise typer.Exit()
 
         time.sleep(settings.WATCH_TICK)
@@ -319,20 +489,15 @@ def request(
         None,
         help="Compose used to provision system-under-test. If not set, tests will expect 'container' provision method specified in tmt plans.",  # noqa
     ),
-    hardware: List[str] = typer.Option(
-        None,
-        help=(
-            "HW requirements, expressed as key/value pairs. Keys can consist of several properties, "
-            "e.g. ``disk.space='>= 40 GiB'``, such keys will be merged in the resulting environment "
-            "with other keys sharing the path: ``cpu.family=79`` and ``cpu.model=6`` would be merged, "
-            "not overwriting each other. See https://tmt.readthedocs.io/en/stable/spec/hardware.html "
-            "for the hardware specification."
-        ),
-    ),
+    hardware: List[str] = OPTION_HARDWARE,
     kickstart: Optional[List[str]] = OPTION_KICKSTART,
     pool: Optional[str] = OPTION_POOL,
     cli_tmt_context: Optional[List[str]] = typer.Option(
-        None, "-c", "--context", metavar="key=value", help="Context variables to pass to `tmt`."
+        None,
+        "-c",
+        "--context",
+        metavar="key=value|@file",
+        help="Context variables to pass to `tmt`. The @ prefix marks a yaml file to load.",
     ),
     variables: Optional[List[str]] = OPTION_VARIABLES,
     secrets: Optional[List[str]] = OPTION_SECRETS,
@@ -340,10 +505,11 @@ def request(
         None,
         "-T",
         "--tmt-environment",
-        metavar="key=value",
+        metavar="key=value|@file",
         help=(
             "Environment variables to pass to the tmt process. "
-            "Used to configure tmt report plugins like reportportal or polarion."
+            "Used to configure tmt report plugins like reportportal or polarion. "
+            "The @ prefix marks a yaml file to load."
         ),
     ),
     no_wait: bool = typer.Option(False, help="Skip waiting for request completion."),
@@ -354,7 +520,11 @@ def request(
     repository: List[str] = OPTION_REPOSITORY,
     repository_file: List[str] = OPTION_REPOSITORY_FILE,
     tags: Optional[List[str]] = typer.Option(
-        None, "-t", "--tag", metavar="key=value", help="Tag cloud resources with given value."
+        None,
+        "-t",
+        "--tag",
+        metavar="key=value|@file",
+        help="Tag cloud resources with given value. The @ prefix marks a yaml file to load.",
     ),
     watchdog_dispatch_delay: Optional[int] = typer.Option(
         None,
@@ -603,7 +773,7 @@ def request(
         exit_error(f"Unexpected error. Please file an issue to {settings.ISSUE_TRACKER}.")
 
     # watch
-    watch(api_url, response.json()['id'], no_wait)
+    watch(api_url, response.json()['id'], no_wait, format=WatchFormat.text)
 
 
 def restart(
@@ -627,16 +797,7 @@ def restart(
     git_url: Optional[str] = typer.Option(None, help="Force URL of the GIT repository to test."),
     git_ref: Optional[str] = typer.Option(None, help="Force GIT ref or branch to test."),
     git_merge_sha: Optional[str] = typer.Option(None, help="Force GIT ref or branch into which --ref will be merged."),
-    hardware: List[str] = typer.Option(
-        None,
-        help=(
-            "HW requirements, expressed as key/value pairs. Keys can consist of several properties, "
-            "e.g. ``disk.space='>= 40 GiB'``, such keys will be merged in the resulting environment "
-            "with other keys sharing the path: ``cpu.family=79`` and ``cpu.model=6`` would be merged, "
-            "not overwriting each other. See https://tmt.readthedocs.io/en/stable/spec/hardware.html "
-            "for the hardware specification."
-        ),
-    ),
+    hardware: List[str] = OPTION_HARDWARE,
     tmt_plan_name: Optional[str] = OPTION_TMT_PLAN_NAME,
     tmt_plan_filter: Optional[str] = OPTION_TMT_PLAN_FILTER,
     tmt_test_name: Optional[str] = OPTION_TMT_TEST_NAME,
@@ -806,7 +967,7 @@ def restart(
         exit_error(f"Unexpected error. Please file an issue to {settings.ISSUE_TRACKER}.")
 
     # watch
-    watch(str(api_url), response.json()['id'], no_wait)
+    watch(str(api_url), response.json()['id'], no_wait, format=WatchFormat.text)
 
 
 def run(
@@ -894,6 +1055,8 @@ def run(
     if verbose:
         console.print(f"🔎 api [blue]{get_url}[/blue]")
 
+    search: Optional[re.Match[str]] = None
+
     # wait for the sanity test to finish
     with Progress(
         SpinnerColumn(),
@@ -941,6 +1104,12 @@ def run(
         try:
             search = re.search(r'href="(.*)" name="workdir"', session.get(f"{artifacts_url}/results.xml").text)
 
+        except requests.exceptions.SSLError:
+            console.print(
+                "\r🚫 [yellow]artifacts unreachable via SSL, do you have RH CA certificates installed?[/yellow]"
+            )
+            console.print(f"\r🚢 artifacts [blue]{artifacts_url}[/blue]")
+
         except requests.exceptions.ConnectionError:
             console.print("\r🚫 [yellow]artifacts unreachable, are you on VPN?[/yellow]")
             console.print(f"\r🚢 artifacts [blue]{artifacts_url}[/blue]")
@@ -949,7 +1118,6 @@ def run(
     if not search:
         exit_error("Could not find working directory, cannot continue")
 
-    assert search
     workdir = str(search.groups(1)[0])
     output = f"{workdir}/testing-farm/sanity/execute/data/guest/default-0/testing-farm/script-1/output.txt"
 
@@ -1186,12 +1354,24 @@ def reserve(
                 if not pipeline_log:
                     exit_error(f"Pipeline log was empty. Please file an issue to {settings.ISSUE_TRACKER}.")
 
+            except requests.exceptions.SSLError:
+                exit_error(
+                    textwrap.dedent(
+                        f"""
+                    Failed to access Testing Farm artifacts because of SSL validation error.
+                    If you use Red Hat Ranch please make sure you have Red Hat CA certificates installed.
+                    Otherwise file an issue to {settings.ISSUE_TRACKER}.
+                """
+                    )
+                )
+                return
+
             except requests.exceptions.ConnectionError:
                 exit_error(
                     textwrap.dedent(
                         f"""
                     Failed to access Testing Farm artifacts.
-                    If you use Red Hat Ranch please make sure you are conneted to the VPN.
+                    If you use Red Hat Ranch please make sure you are connected to the VPN.
                     Otherwise file an issue to {settings.ISSUE_TRACKER}.
                 """
                     )

{tft_cli-0.0.17 → tft_cli-0.0.18}/src/tft/cli/config.py

@@ -19,4 +19,6 @@ settings = LazySettings(
     DEFAULT_API_RETRIES=7,
     # should lead to delays of 0.5, 1, 2, 4, 8, 16, 32 seconds
     DEFAULT_RETRY_BACKOFF_FACTOR=1,
+    # system CA certificates path, default for RHEL variants
+    REQUESTS_CA_BUNDLE="/etc/ssl/certs/ca-bundle.crt",
 )

{tft_cli-0.0.17 → tft_cli-0.0.18}/src/tft/cli/tool.py

@@ -21,3 +21,8 @@ app.command()(commands.watch)
 # This command is available only for the container based deployment
 if os.path.exists(settings.CONTAINER_SIGN):
     app.command()(commands.update)
+
+# Expose REQUESTS_CA_BUNDLE in the environment for RHEL-like systems
+# This is needed for custom CA certificates to nicely work.
+if "REQUESTS_CA_BUNDLE" not in os.environ and os.path.exists(settings.REQUESTS_CA_BUNDLE):
+    os.environ["REQUESTS_CA_BUNDLE"] = settings.REQUESTS_CA_BUNDLE

{tft_cli-0.0.17 → tft_cli-0.0.18}/src/tft/cli/utils.py

@@ -4,21 +4,24 @@
 import glob
 import os
 import subprocess
+import sys
 import uuid
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, NoReturn, Optional, Union
 
 import requests
 import requests.adapters
 import typer
 from rich.console import Console
+from ruamel.yaml import YAML
 from urllib3 import Retry
 
 from tft.cli.config import settings
 
 console = Console(soft_wrap=True)
+console_stderr = Console(soft_wrap=True, file=sys.stderr)
 
 
-def exit_error(error: str):
+def exit_error(error: str) -> NoReturn:
     """Exit with given error message"""
     console.print(f"⛔ {error}", style="red")
     raise typer.Exit(code=255)
@@ -91,15 +94,46 @@ def hw_constraints(hardware: List[str]) -> Dict[Any, Any]:
     return {key: value if key not in ("disk", "network") else [value] for key, value in constraints.items()}
 
 
+def options_from_file(filepath) -> Dict[str, str]:
+    """Read environment variables from a yaml file."""
+
+    with open(filepath, 'r') as file:
+        try:
+            yaml = YAML(typ="safe").load(file.read())
+        except Exception:
+            exit_error(f"Failed to load variables from yaml file {filepath}.")
+
+        if not yaml:  # pyre-ignore[61]  # pyre ignores NoReturn in exit_error
+            return {}
+
+        if not isinstance(yaml, dict):  # pyre-ignore[61]  # pyre ignores NoReturn in exit_error
+            exit_error(f"Environment file {filepath} is not a dict.")
+
+        if any([isinstance(value, (list, dict)) for value in yaml.values()]):
+            exit_error(f"Values of environment file {filepath} are not primitive types.")
+
+        return yaml  # pyre-ignore[61]  # pyre ignores NoReturn in exit_error
+
+
 def options_to_dict(name: str, options: List[str]) -> Dict[str, str]:
-    """Create a dictionary from list of `key=value` options"""
-    try:
-        return {option.split("=", 1)[0]: option.split("=", 1)[1] for option in options}
+    """Create a dictionary from list of `key=value|@file` options"""
 
-    except IndexError:
-        exit_error(f"Options for {name} are invalid, must be defined as `key=value`")
+    options_dict = {}
+    for option in options:
+        # Option is `@file`
+        if option.startswith('@'):
+            if not os.path.isfile(option[1:]):
+                exit_error(f"Invalid environment file in option `{option}` specified.")
+            options_dict.update(options_from_file(option[1:]))
+
+        # Option is `key=value`
+        else:
+            try:
+                options_dict.update({option.split("=", 1)[0]: option.split("=", 1)[1]})
+            except IndexError:
+                exit_error(f"Option `{option}` is invalid, must be defined as `key=value|@file`.")
 
-    return {}
+    return options_dict
 
 
 def uuid_valid(value: str, version: int = 4) -> bool: