tft-cli 0.0.22__py3-none-any.whl → 0.0.24__py3-none-any.whl

tft/cli/commands.py

@@ -19,7 +19,8 @@ from typing import Any, Dict, List, Optional
 import pkg_resources
 import requests
 import typer
-from rich import print
+from click.core import ParameterSource  # pyre-ignore[21]
+from rich import print, print_json
 from rich.progress import Progress, SpinnerColumn, TextColumn
 from rich.table import Table
 
@@ -40,13 +41,14 @@ from tft.cli.utils import (
 
 cli_version: str = pkg_resources.get_distribution("tft-cli").version
 
-TestingFarmRequestV1: Dict[str, Any] = {'api_key': None, 'test': {}, 'environments': None}
+TestingFarmRequestV1: Dict[str, Any] = {'test': {}, 'environments': None}
 Environment: Dict[str, Any] = {'arch': None, 'os': None, 'pool': None, 'artifacts': None, 'variables': {}}
 TestTMT: Dict[str, Any] = {'url': None, 'ref': None, 'name': None}
 TestSTI: Dict[str, Any] = {'url': None, 'ref': None}
 
 REQUEST_PANEL_TMT = "TMT Options"
 REQUEST_PANEL_STI = "STI Options"
+REQUEST_PANEL_RESERVE = "Reserve Options"
 
 RESERVE_PANEL_GENERAL = "General Options"
 RESERVE_PANEL_ENVIRONMENT = "Environment Options"
@@ -56,8 +58,10 @@ RUN_REPO = "https://gitlab.com/testing-farm/tests"
 RUN_PLAN = "/testing-farm/sanity"
 
 RESERVE_PLAN = os.getenv("TESTING_FARM_RESERVE_PLAN", "/testing-farm/reserve")
+RESERVE_TEST = os.getenv("TESTING_FARM_RESERVE_TEST", "/testing-farm/reserve-system")
 RESERVE_URL = os.getenv("TESTING_FARM_RESERVE_URL", "https://gitlab.com/testing-farm/tests")
 RESERVE_REF = os.getenv("TESTING_FARM_RESERVE_REF", "main")
+RESERVE_TMT_DISCOVER_EXTRA_ARGS = f"--insert --how fmf --url {RESERVE_URL} --ref {RESERVE_REF} --test {RESERVE_TEST}"
 
 DEFAULT_PIPELINE_TIMEOUT = 60 * 12
 
@@ -174,10 +178,14 @@ OPTION_POOL: Optional[str] = typer.Option(
     rich_help_panel=RESERVE_PANEL_ENVIRONMENT,
 )
 OPTION_REDHAT_BREW_BUILD: List[str] = typer.Option(
-    None, help="Brew build task IDs to install on the test environment.", rich_help_panel=RESERVE_PANEL_ENVIRONMENT
+    None,
+    help="Brew build task IDs or build NVRs to install on the test environment.",
+    rich_help_panel=RESERVE_PANEL_ENVIRONMENT,
 )
 OPTION_FEDORA_KOJI_BUILD: List[str] = typer.Option(
-    None, help="Koji build task IDs to install on the test environment.", rich_help_panel=RESERVE_PANEL_ENVIRONMENT
+    None,
+    help="Koji build task IDs or build NVRs to install on the test environment.",
+    rich_help_panel=RESERVE_PANEL_ENVIRONMENT,
 )
 OPTION_FEDORA_COPR_BUILD: List[str] = typer.Option(
     None,
@@ -241,6 +249,220 @@ OPTION_TAGS = typer.Option(
     metavar="key=value|@file",
     help="Tag cloud resources with given value. The @ prefix marks a yaml file to load.",
 )
+OPTION_RESERVE: bool = typer.Option(
+    False,
+    help="Reserve machine after testing, similarly to the `reserve` command.",
+    rich_help_panel=REQUEST_PANEL_RESERVE,
+)
+
+
+def _option_autoconnect(panel: str) -> bool:
+    return typer.Option(True, help="Automatically connect to the guest via SSH.", rich_help_panel=panel)
+
+
+def _option_ssh_public_keys(panel: str) -> List[str]:
+    return typer.Option(
+        ["~/.ssh/*.pub"],
+        "--ssh-public-key",
+        help="Path to SSH public key(s) used to connect. Supports globbing.",
+        rich_help_panel=panel,
+    )
+
+
+def _option_reservation_duration(panel: str) -> int:
+    return typer.Option(
+        settings.DEFAULT_RESERVATION_DURATION,
+        "--duration",
+        help="Set the reservation duration in minutes. By default the reservation is for 30 minutes.",
+        rich_help_panel=panel,
+    )
+
+
+def _option_debug_reservation(panel: Optional[str] = None) -> bool:
+    return typer.Option(
+        False,
+        help="Enable debug messages in the reservation code. Useful for testing changes to reservation code.",
+        rich_help_panel=panel,
+    )
+
+
+def _generate_tmt_extra_args(step: str) -> Optional[List[str]]:
+    return typer.Option(
+        None,
+        help=(
+            f"Additional options passed to the \"{step}\" step. "
+            "Can be specified multiple times for multiple additions."
+        ),
+        rich_help_panel=REQUEST_PANEL_TMT,
+    )
+
+
+def _sanity_reserve() -> None:
+    """
+    Sanity checks for reservation support.
+    """
+
+    # Check of SSH_AUTH_SOCK is defined
+    ssh_auth_sock = os.getenv("SSH_AUTH_SOCK")
+    if not ssh_auth_sock:
+        exit_error(
+            "No 'ssh-agent' seems to be running, it is required for reservations to work, cannot continue.\n"
+            "SSH_AUTH_SOCK is not defined, make sure the ssh-agent is running by executing 'eval `ssh-agent`'."
+        )
+
+    # Check if SSH_AUTH_SOCK exists
+    if not os.path.exists(ssh_auth_sock):
+        exit_error(
+            "SSH_AUTH_SOCK socket does not exist, make sure the ssh-agent is running by executing 'eval `ssh-agent`'."
+        )
+
+    # Check if value of SSH_AUTH_SOCK is socket
+    if not stat.S_ISSOCK(os.stat(ssh_auth_sock).st_mode):
+        exit_error("SSH_AUTH_SOCK is not a socket, make sure the ssh-agent is running by executing 'eval `ssh-agent`'.")
+
+    # Check if ssh-add -L is not empty
+    ssh_add_output = subprocess.run(["ssh-add", "-L"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    if ssh_add_output.returncode != 0:
+        exit_error("No SSH identities found in the SSH agent. Please run `ssh-add`.")
+
+
+def _handle_reservation(session, request_id: str, autoconnect: bool = False) -> None:
+    """
+    Handle the reservation for :py:func:``request`` and :py:func:``restart`` commands.
+    """
+    # Get artifacts url
+    request_url = urllib.parse.urljoin(settings.API_URL, f"/v0.1/requests/{request_id}")
+    response = session.get(request_url)
+    artifacts_url = response.json()['run']['artifacts']
+
+    try:
+        pipeline_log = session.get(f"{artifacts_url}/pipeline.log").text
+
+        if not pipeline_log:
+            exit_error(f"Pipeline log was empty. Please file an issue to {settings.ISSUE_TRACKER}.")
+
+    except requests.exceptions.SSLError:
+        exit_error(
+            textwrap.dedent(
+                f"""
+            Failed to access Testing Farm artifacts because of SSL validation error.
+            If you use Red Hat Ranch please make sure you have Red Hat CA certificates installed.
+            Otherwise file an issue to {settings.ISSUE_TRACKER}.
+        """
+            )
+        )
+        return
+
+    except requests.exceptions.ConnectionError:
+        exit_error(
+            textwrap.dedent(
+                f"""
+            Failed to access Testing Farm artifacts.
+            If you use Red Hat Ranch please make sure you are connected to the VPN.
+            Otherwise file an issue to {settings.ISSUE_TRACKER}.
+        """
+            )
+        )
+        return
+
+    # match any hostname or IP address from gluetool modules log
+    guests = re.findall(r'Guest is ready.*root@([\d\w\.-]+)', pipeline_log)
+
+    if not guests:
+        exit_error(
+            textwrap.dedent(
+                f"""
+            No guests found to connect to. This is unexpected, please file an issue
+            to {settings.ISSUE_TRACKER}.
+        """
+            )
+        )
+
+    if len(guests) > 1:
+        for guest in guests:
+            console.print(f"🌎 ssh root@{guest}")
+        return
+    else:
+        console.print(f"🌎 ssh root@{guests[0]}")
+
+    if autoconnect:
+        os.system(f"ssh -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null root@{guests[0]}")  # noqa: E501
+
+
+def _localhost_ingress_rule(session: requests.Session) -> str:
+    try:
+        get_ip = session.get(settings.PUBLIC_IP_CHECKER_URL)
+    except requests.exceptions.RequestException as err:
+        exit_error(f"Could not get workstation ip to form a security group rule: {err}")
+
+    if get_ip.ok:
+        ip = get_ip.text.strip()
+        return f"-1:{ip}:-1"
+
+    else:
+        exit_error(f"Got {get_ip.status_code} while checking {settings.PUBLIC_IP_CHECKER_URL}")
+
+
+def _add_reservation(
+    ssh_public_keys: List[str],
+    rules: Dict[str, Any],
+    duration: int,
+    environment: Dict[str, Any],
+    debug_reservation: bool,
+):
+    """
+    Add discovery of the reservation test to the given environment.
+    """
+    authorized_keys = read_glob_paths(ssh_public_keys).encode("utf-8")
+    if not authorized_keys:
+        exit_error(f"No public SSH keys found under {', '.join(ssh_public_keys)}, cannot continue.")
+
+    authorized_keys_bytes = base64.b64encode(authorized_keys)
+
+    if "secrets" not in environment or environment["secrets"] is None:
+        environment["secrets"] = {}
+
+    environment["secrets"].update({"TF_RESERVATION_AUTHORIZED_KEYS_BASE64": authorized_keys_bytes.decode("utf-8")})
+
+    if "settings" not in environment or environment["settings"] is None:
+        environment["settings"] = {}
+
+    if "provisioning" not in environment["settings"] or environment["settings"]["provisioning"] is None:
+        environment["settings"]["provisioning"] = {}
+
+    environment["settings"]["provisioning"].update(rules)
+
+    if "variables" not in environment or environment["variables"] is None:
+        environment["variables"] = {}
+
+    environment["variables"].update({"TF_RESERVATION_DURATION": str(duration)})
+
+    if debug_reservation:
+        environment["variables"].update({"TF_RESERVATION_DEBUG": "1"})
+
+    if "tmt" not in environment or environment["tmt"] is None:
+        environment["tmt"] = {"extra_args": {}}
+
+    if "extra_args" not in environment["tmt"] or environment["tmt"]["extra_args"] is None:
+        environment["tmt"]["extra_args"] = {}
+
+    if "discover" not in environment["tmt"]["extra_args"] or environment["tmt"]["extra_args"]["discover"] is None:
+        environment["tmt"]["extra_args"]["discover"] = []
+
+    # add reservation if not already present
+    if RESERVE_TMT_DISCOVER_EXTRA_ARGS not in environment["tmt"]["extra_args"]["discover"]:
+        environment["tmt"]["extra_args"]["discover"].append(RESERVE_TMT_DISCOVER_EXTRA_ARGS)
+
+
+def _contains_compose(environments: List[Dict[str, Any]]):
+    """
+    Returns true if any of environments has ``os.compose`` defined.
+    """
+    for environment in environments:
+        if "os" in environment and environment["os"]:
+            if "compose" in environment["os"] and environment["os"]["compose"]:
+                return True
+    return False
 
 
 # NOTE(ivasilev) Largely borrowed from artemis-cli
@@ -288,6 +510,14 @@ def _parse_security_group_rules(ingress_rules: List[str], egress_rules: List[str
     return security_group_rules
 
 
+def _get_headers(api_key: str) -> Dict[str, str]:
+    """
+    Return a dict with headers for a request to Testing Farm API.
+    Used for authentication.
+    """
+    return {'Authorization': f'Bearer {api_key}'}
+
+
 def _parse_xunit(xunit: str):
     """
     A helper that parses xunit file into sets of passed_plans/failed_plans/errored_plans per arch.
@@ -307,13 +537,14 @@ def _parse_xunit(xunit: str):
 
     failed_plans = {}
     passed_plans = {}
+    skipped_plans = {}
     errored_plans = {}
 
     results_root = ET.fromstring(xunit)
     for plan in results_root.findall('./testsuite'):
-        # Try to get information about the environment (stored under testcase/testing-environment), may be
+        # Try to get information about the environment (stored under ./testing-environment), may be
         # absent if state is undefined
-        testing_environment: Optional[ET.Element] = plan.find('./testcase/testing-environment[@name="requested"]')
+        testing_environment: Optional[ET.Element] = plan.find('./testing-environment[@name="requested"]')
         if not testing_environment:
             console_stderr.print(
                 f'Could not find env specifications for {plan.get("name")}, assuming fail for all arches'
@@ -331,13 +562,15 @@ def _parse_xunit(xunit: str):
             _add_plan(passed_plans, arch, plan)
         elif plan.get('result') == 'failed':
             _add_plan(failed_plans, arch, plan)
+        elif plan.get('result') == 'skipped':
+            _add_plan(skipped_plans, arch, plan)
         else:
             _add_plan(errored_plans, arch, plan)
 
     # Let's remove possible duplicates among N/A errored out tests
     if 'N/A' in errored_plans:
         errored_plans['N/A'] = list(set(errored_plans['N/A']))
-    return passed_plans, failed_plans, errored_plans
+    return passed_plans, failed_plans, skipped_plans, errored_plans
 
 
 def _get_request_summary(request: dict, session: requests.Session):
@@ -355,7 +588,7 @@ def _get_request_summary(request: dict, session: requests.Session):
                 xunit = response.text
         except requests.exceptions.ConnectionError:
             console_stderr.print("Could not get xunit results")
-    passed_plans, failed_plans, errored_plans = _parse_xunit(xunit)
+    passed_plans, failed_plans, skipped_plans, errored_plans = _parse_xunit(xunit)
     overall = (request.get("result") or {}).get("overall")
     arches_requested = [env['arch'] for env in request['environments_requested']]
 
@@ -367,6 +600,7 @@ def _get_request_summary(request: dict, session: requests.Session):
         'arches_requested': arches_requested,
         'errored_plans': errored_plans,
         'failed_plans': failed_plans,
+        'skipped_plans': skipped_plans,
         'passed_plans': passed_plans,
     }
 
@@ -385,6 +619,7 @@ def _print_summary_table(summary: dict, format: Optional[WatchFormat], show_deta
     # Let's transform plans maps into collection of plans to display plan result per arch statistics
     errored = _get_plans_list(summary['errored_plans'])
     failed = _get_plans_list(summary['failed_plans'])
+    skipped = _get_plans_list(summary['skipped_plans'])
     passed = _get_plans_list(summary['passed_plans'])
     generic_info_table = Table(show_header=True, header_style="bold magenta")
     arches_requested = summary['arches_requested']
@@ -399,11 +634,12 @@ def _print_summary_table(summary: dict, format: Optional[WatchFormat], show_deta
         ','.join(arches_requested),
         str(len(errored)),
         str(len(failed)),
+        str(len(skipped)),
         str(len(passed)),
     )
     console.print(generic_info_table)
 
-    all_plans = sorted(set(errored + failed + passed))
+    all_plans = sorted(set(errored + failed + skipped + passed))
     details_table = Table(show_header=True, header_style="bold magenta")
     for column in ["plan"] + arches_requested:
         details_table.add_column(column)
@@ -413,6 +649,8 @@ def _print_summary_table(summary: dict, format: Optional[WatchFormat], show_deta
         for arch in arches_requested:
             if _has_plan(summary['passed_plans'], arch, plan):
                 res = '[green]pass[/green]'
+            elif _has_plan(summary['skipped_plans'], arch, plan):
+                res = '[white]skip[/white]'
             elif _has_plan(summary['failed_plans'], arch, plan):
                 res = '[red]fail[/red]'
             elif _has_plan(summary['errored_plans'], 'N/A', plan):
@@ -432,6 +670,8 @@ def watch(
     id: str = typer.Option(..., help="Request ID to watch"),
     no_wait: bool = typer.Option(False, help="Skip waiting for request completion."),
     format: Optional[WatchFormat] = typer.Option(WatchFormat.text, help="Output format"),
+    autoconnect: bool = typer.Option(True, hidden=True),
+    reserve: bool = typer.Option(False, hidden=True),
 ):
     def _console_print(*args, **kwargs):
         """A helper function that will skip printing to console if output format is json"""
@@ -458,6 +698,24 @@ def watch(
     session = requests.Session()
     install_http_retries(session)
 
+    def _is_reserved(session, request):
+        artifacts_url = (request.get('run') or {}).get('artifacts')
+
+        if not artifacts_url:
+            return False
+
+        try:
+            workdir = re.search(r'href="(.*)" name="workdir"', session.get(f"{artifacts_url}/results.xml").text)
+        except requests.exceptions.SSLError:
+            exit_error("Artifacts unreachable via SSL, do you have RH CA certificates installed?[/yellow]")
+
+        if workdir:
+            # finish early if reservation is running
+            if re.search(r"\[\+\] Reservation tick:", session.get(f"{workdir.group(1)}/log.txt").text):
+                return True
+
+        return False
+
     while True:
         try:
             response = session.get(get_url)
@@ -477,6 +735,11 @@ def watch(
         state = request["state"]
 
         if state == current_state:
+            # check for reservation status and finish early if reserved
+            if reserve and _is_reserved(session, request):
+                _handle_reservation(session, request["id"], autoconnect)
+                return
+
             time.sleep(1)
             continue
 
@@ -524,6 +787,10 @@ def watch(
             _print_summary_table(request_summary, format)
             raise typer.Exit(code=2)
 
+        elif state in ["canceled", "cancel-requested"]:
+            _console_print("⚠️ pipeline cancelled", style="yellow")
+            raise typer.Exit(code=3)
+
         if no_wait:
             _print_summary_table(request_summary, format, show_details=False)
             raise typer.Exit()
@@ -539,7 +806,7 @@ def version():
 def request(
     api_url: str = ARGUMENT_API_URL,
     api_token: str = ARGUMENT_API_TOKEN,
-    timeout: Optional[int] = typer.Option(
+    timeout: int = typer.Option(
         DEFAULT_PIPELINE_TIMEOUT,
         help="Set the timeout for the request in minutes. If the test takes longer than this, it will be terminated.",
     ),
@@ -624,6 +891,14 @@ def request(
         None, help="URL of the icon of the user's webpage. It will be shown in the results viewer."
     ),
     parallel_limit: Optional[int] = OPTION_PARALLEL_LIMIT,
+    tmt_discover: Optional[List[str]] = _generate_tmt_extra_args("discover"),
+    tmt_prepare: Optional[List[str]] = _generate_tmt_extra_args("prepare"),
+    tmt_finish: Optional[List[str]] = _generate_tmt_extra_args("finish"),
+    reserve: bool = OPTION_RESERVE,
+    ssh_public_keys: List[str] = _option_ssh_public_keys(REQUEST_PANEL_RESERVE),
+    autoconnect: bool = _option_autoconnect(REQUEST_PANEL_RESERVE),
+    reservation_duration: int = _option_reservation_duration(REQUEST_PANEL_RESERVE),
+    debug_reservation: bool = _option_debug_reservation(REQUEST_PANEL_RESERVE),
 ):
     """
     Request testing from Testing Farm.
@@ -653,6 +928,9 @@ def request(
         git_url = str(settings.TESTING_FARM_TESTS_GIT_URL)
         tmt_plan_name = str(settings.TESTING_FARM_SANITY_PLAN)
 
+    if reserve:
+        _sanity_reserve()
+
     # resolve git repository details from the current repository
     if not git_url:
         if not git_available:
@@ -687,7 +965,7 @@ def request(
             git_ref = cmd_output_or_exit("git rev-parse HEAD", "could not autodetect git ref")
 
         # detect test type from local files
-        if os.path.exists(".fmf/version"):
+        if os.path.exists(os.path.join((tmt_path or ""), ".fmf/version")):
             test_type = "fmf"
         elif os.path.exists("tests/tests.yml"):
             test_type = "sti"
@@ -780,8 +1058,46 @@ def request(
         if tmt_environment:
             environment["tmt"].update({"environment": options_to_dict("tmt environment variables", tmt_environment)})
 
+        if tmt_discover or tmt_prepare or tmt_finish:
+            if "extra_args" not in environment["tmt"]:
+                environment["tmt"]["extra_args"] = {}
+
+        if tmt_discover:
+            environment["tmt"]["extra_args"]["discover"] = tmt_discover
+
+        if tmt_prepare:
+            environment["tmt"]["extra_args"]["prepare"] = tmt_prepare
+
+        if tmt_finish:
+            environment["tmt"]["extra_args"]["finish"] = tmt_finish
+
         environments.append(environment)
 
+    # Setting up retries
+    session = requests.Session()
+    install_http_retries(session)
+
+    if reserve:
+        if not _contains_compose(environments):
+            exit_error("Reservations are not supported with container executions, cannot continue")
+
+        if len(environments) > 1:
+            exit_error("Reservations are currently supported for a single plan, cannot continue")
+
+        rules = _parse_security_group_rules([_localhost_ingress_rule(session)], [])
+
+        for environment in environments:
+            _add_reservation(
+                ssh_public_keys=ssh_public_keys,
+                rules=rules,
+                duration=reservation_duration,
+                environment=environment,
+                debug_reservation=debug_reservation,
+            )
+
+        machine_pre = "Machine" if len(environments) == 1 else str(len(environments)) + " machines"
+        console.print(f"🛟 {machine_pre} will be reserved after testing")
+
     if any(
         provisioning_detail
         for provisioning_detail in [
@@ -817,7 +1133,6 @@ def request(
 
     # create final request
     request = TestingFarmRequestV1
-    request["api_key"] = api_token
     if test_type == "fmf":
         test["path"] = tmt_path
         request["test"]["fmf"] = test
@@ -826,7 +1141,23 @@ def request(
 
     request["environments"] = environments
     request["settings"] = {}
-    request["settings"]["pipeline"] = {"timeout": timeout}
+
+    if reserve or pipeline_type or parallel_limit or timeout != DEFAULT_PIPELINE_TIMEOUT:
+        request["settings"]["pipeline"] = {}
+
+    # in case the reservation duration is more than the pipeline timeout, adjust also the pipeline timeout
+    if reserve:
+        if reservation_duration > timeout:
+            request["settings"]["pipeline"] = {"timeout": reservation_duration}
+            console.print(f"⏳ Maximum reservation time is {reservation_duration} minutes")
+        else:
+            request["settings"]["pipeline"] = {"timeout": timeout}
+            console.print(f"⏳ Maximum reservation time is {timeout} minutes")
+
+    # forced pipeline timeout
+    elif timeout != DEFAULT_PIPELINE_TIMEOUT:
+        console.print(f"⏳ Pipeline timeout forced to {timeout} minutes")
+        request["settings"]["pipeline"] = {"timeout": timeout}
 
     if pipeline_type:
         request["settings"]["pipeline"]["type"] = pipeline_type.value
@@ -849,18 +1180,14 @@ def request(
     # submit request to Testing Farm
     post_url = urllib.parse.urljoin(api_url, "v0.1/requests")
 
-    # Setting up retries
-    session = requests.Session()
-    install_http_retries(session)
-
     # dry run
     if dry_run:
         console.print("🔍 Dry run, showing POST json only", style="bright_yellow")
-        print(json.dumps(request, indent=4, separators=(',', ': ')))
+        print_json(json.dumps(request, indent=4, separators=(',', ': ')))
         raise typer.Exit()
 
     # handle errors
-    response = session.post(post_url, json=request)
+    response = session.post(post_url, json=request, headers=_get_headers(api_token))
     if response.status_code == 401:
         exit_error(f"API token is invalid. See {settings.ONBOARDING_DOCS} for more information.")
 
@@ -874,11 +1201,14 @@ def request(
         print(response.text)
         exit_error(f"Unexpected error. Please file an issue to {settings.ISSUE_TRACKER}.")
 
-    # watch
-    watch(api_url, response.json()['id'], no_wait, format=WatchFormat.text)
+    request_id = response.json()['id']
+
+    # Watch the request and handle reservation
+    watch(api_url, request_id, no_wait, reserve=reserve, autoconnect=autoconnect, format=WatchFormat.text)
 
 
 def restart(
+    context: typer.Context,
     request_id: str = typer.Argument(..., help="Testing Farm request ID or a string containing it."),
     api_url: str = ARGUMENT_API_URL,
     internal_api_url: str = typer.Argument(
@@ -905,12 +1235,20 @@ def restart(
     tmt_plan_filter: Optional[str] = OPTION_TMT_PLAN_FILTER,
     tmt_test_name: Optional[str] = OPTION_TMT_TEST_NAME,
     tmt_test_filter: Optional[str] = OPTION_TMT_TEST_FILTER,
-    tmt_path: str = OPTION_TMT_PATH,
+    tmt_path: Optional[str] = OPTION_TMT_PATH,
+    tmt_discover: Optional[List[str]] = _generate_tmt_extra_args("discover"),
+    tmt_prepare: Optional[List[str]] = _generate_tmt_extra_args("prepare"),
+    tmt_finish: Optional[List[str]] = _generate_tmt_extra_args("finish"),
     worker_image: Optional[str] = OPTION_WORKER_IMAGE,
     no_wait: bool = typer.Option(False, help="Skip waiting for request completion."),
     dry_run: bool = OPTION_DRY_RUN,
     pipeline_type: Optional[PipelineType] = OPTION_PIPELINE_TYPE,
     parallel_limit: Optional[int] = OPTION_PARALLEL_LIMIT,
+    reserve: bool = OPTION_RESERVE,
+    ssh_public_keys: List[str] = _option_ssh_public_keys(REQUEST_PANEL_RESERVE),
+    autoconnect: bool = _option_autoconnect(REQUEST_PANEL_RESERVE),
+    reservation_duration: int = _option_reservation_duration(REQUEST_PANEL_RESERVE),
+    debug_reservation: bool = _option_debug_reservation(REQUEST_PANEL_RESERVE),
 ):
     """
     Restart a Testing Farm request.
@@ -932,14 +1270,14 @@ def restart(
     _request_id = uuid_match.group()
 
     # Construct URL to the internal API
-    get_url = urllib.parse.urljoin(str(internal_api_url), f"v0.1/requests/{_request_id}?api_key={api_token}")
+    get_url = urllib.parse.urljoin(str(internal_api_url), f"v0.1/requests/{_request_id}")
 
     # Setting up retries
     session = requests.Session()
     install_http_retries(session)
 
     # Get the request details
-    response = session.get(get_url)
+    response = session.get(get_url, headers=_get_headers(api_token))
 
     if response.status_code == 401:
         exit_error(f"API token is invalid. See {settings.ONBOARDING_DOCS} for more information.")
@@ -1018,6 +1356,25 @@ def restart(
         for environment in request['environments']:
             environment["pool"] = pool
 
+    if tmt_discover or tmt_prepare or tmt_finish:
+        for environment in request["environments"]:
+            if "tmt" not in environment:
+                environment["tmt"] = {"extra_args": {}}
+            if "extra_args" not in environment["tmt"]:
+                environment["tmt"]["extra_args"] = {}
+
+    if tmt_discover:
+        for environment in request["environments"]:
+            environment["tmt"]["extra_args"]["discover"] = tmt_discover
+
+    if tmt_prepare:
+        for environment in request["environments"]:
+            environment["tmt"]["extra_args"]["prepare"] = tmt_prepare
+
+    if tmt_finish:
+        for environment in request["environments"]:
+            environment["tmt"]["extra_args"]["finish"] = tmt_finish
+
     test_type = "fmf" if "fmf" in request["test"] else "sti"
 
     if tmt_plan_name:
@@ -1031,7 +1388,9 @@ def restart(
         request["test"][test_type]["plan_filter"] = tmt_plan_filter
 
     if test_type == "fmf":
-        request["test"][test_type]["path"] = tmt_path
+        # The method explained in https://github.com/fastapi/typer/discussions/668
+        if context.get_parameter_source("tmt_path") == ParameterSource.COMMANDLINE:  # pyre-ignore[16]
+            request["test"][test_type]["path"] = tmt_path
 
     # worker image
     if worker_image:
@@ -1041,9 +1400,6 @@ def restart(
         # it is required to have also pipeline key set, otherwise API will fail
         request["settings"]["pipeline"] = request["settings"].get("pipeline", {})
 
-    # Add API key
-    request['api_key'] = api_token
-
     if pipeline_type or parallel_limit:
         if "settings" not in request:
             request["settings"] = {}
@@ -1066,6 +1422,31 @@ def restart(
 
             environment["settings"]["provisioning"]["tags"] = options_to_dict("tags", tags)
 
+    if reserve:
+        if not _contains_compose(request["environments"]):
+            exit_error("Reservations are not supported with container executions, cannot continue")
+
+        if len(request["environments"]) > 1:
+            exit_error("Reservations are currently supported for a single plan, cannot continue")
+
+        rules = _parse_security_group_rules([_localhost_ingress_rule(session)], [])
+
+        for environment in request["environments"]:
+            _add_reservation(
+                ssh_public_keys=ssh_public_keys,
+                rules=rules,
+                duration=reservation_duration,
+                environment=environment,
+                debug_reservation=debug_reservation,
+            )
+
+        machine_pre = (
+            "Machine" if len(request["environments"]) == 1 else str(len(request["environments"])) + " machines"
+        )
+        console.print(
+            f"🕗 {machine_pre} will be reserved after testing for [blue]{str(reservation_duration)}[/blue] minutes"
+        )
+
     # dry run
     if dry_run:
         console.print("🔍 Dry run, showing POST json only", style="bright_yellow")
@@ -1076,7 +1457,7 @@ def restart(
     post_url = urllib.parse.urljoin(str(api_url), "v0.1/requests")
 
     # handle errors
-    response = session.post(post_url, json=request)
+    response = session.post(post_url, json=request, headers=_get_headers(api_token))
     if response.status_code == 401:
         exit_error(f"API token is invalid. See {settings.ONBOARDING_DOCS} for more information.")
 
@@ -1091,7 +1472,9 @@ def restart(
         exit_error(f"Unexpected error. Please file an issue to {settings.ISSUE_TRACKER}.")
 
     # watch
-    watch(str(api_url), response.json()['id'], no_wait, format=WatchFormat.text)
+    watch(
+        str(api_url), response.json()['id'], no_wait, reserve=reserve, autoconnect=autoconnect, format=WatchFormat.text
+    )
 
 
 def run(
@@ -1118,7 +1501,6 @@ def run(
 
     # create request
     request = TestingFarmRequestV1
-    request["api_key"] = settings.API_TOKEN
 
     test = TestTMT
     test["url"] = RUN_REPO
@@ -1162,7 +1544,7 @@ def run(
             raise typer.Exit()
 
     # handle errors
-    response = session.post(post_url, json=request)
+    response = session.post(post_url, json=request, headers=_get_headers(settings.API_TOKEN))
     if response.status_code == 401:
         exit_error(f"API token is invalid. See {settings.ONBOARDING_DOCS} for more information.")
 
@@ -1214,6 +1596,10 @@ def run(
             if state in ["complete", "error"]:
                 break
 
+            if state in ["canceled", "cancel-requested"]:
+                progress.stop()
+                exit_error("Request canceled.")
+
             time.sleep(1)
 
         # workaround TFT-1690
@@ -1254,18 +1640,8 @@ def run(
 
 
 def reserve(
-    ssh_public_keys: List[str] = typer.Option(
-        ["~/.ssh/*.pub"],
-        "--ssh-public-key",
-        help="Path to SSH public key(s) used to connect. Supports globbing.",
-        rich_help_panel=RESERVE_PANEL_GENERAL,
-    ),
-    reservation_duration: int = typer.Option(
-        30,
-        "--duration",
-        help="Set the reservation duration in minutes. By default the reservation is for 30 minutes.",
-        rich_help_panel=RESERVE_PANEL_GENERAL,
-    ),
+    ssh_public_keys: List[str] = _option_ssh_public_keys(RESERVE_PANEL_GENERAL),
+    reservation_duration: int = _option_reservation_duration(RESERVE_PANEL_GENERAL),
     arch: str = typer.Option(
         "x86_64", help="Hardware platform of the system to be provisioned.", rich_help_panel=RESERVE_PANEL_ENVIRONMENT
     ),
@@ -1290,15 +1666,17 @@ def reserve(
         help="Output only the request ID.",
         rich_help_panel=RESERVE_PANEL_OUTPUT,
     ),
-    autoconnect: bool = typer.Option(
-        True, help="Automatically connect to the guest via SSH.", rich_help_panel=RESERVE_PANEL_GENERAL
-    ),
+    autoconnect: bool = _option_autoconnect(RESERVE_PANEL_GENERAL),
     worker_image: Optional[str] = OPTION_WORKER_IMAGE,
     security_group_rule_ingress: Optional[List[str]] = OPTION_SECURITY_GROUP_RULE_INGRESS,
     security_group_rule_egress: Optional[List[str]] = OPTION_SECURITY_GROUP_RULE_EGRESS,
     skip_workstation_access: bool = typer.Option(
         False, help="Do not allow ingress traffic from this workstation's ip to the reserved machine"
     ),
+    git_ref: Optional[str] = typer.Option(
+        None, help="Force GIT ref or branch. Useful for testing changes to reservation plan."
+    ),
+    debug_reservation: bool = _option_debug_reservation(),
 ):
     """
     Reserve a system in Testing Farm.
@@ -1308,27 +1686,7 @@ def reserve(
         if not print_only_request_id:
             console.print(message)
 
-    # Sanity checks for ssh-agent
-
-    # Check of SSH_AUTH_SOCK is defined
-    ssh_auth_sock = os.getenv("SSH_AUTH_SOCK")
-    if not ssh_auth_sock:
-        exit_error("SSH_AUTH_SOCK is not defined, make sure the ssh-agent is running by executing 'eval `ssh-agent`'.")
-
-    # Check if SSH_AUTH_SOCK exists
-    if not os.path.exists(ssh_auth_sock):
-        exit_error(
-            "SSH_AUTH_SOCK socket does not exist, make sure the ssh-agent is running by executing 'eval `ssh-agent`'."
-        )
-
-    # Check if value of SSH_AUTH_SOCK is socket
-    if not stat.S_ISSOCK(os.stat(ssh_auth_sock).st_mode):
-        exit_error("SSH_AUTH_SOCK is not a socket, make sure the ssh-agent is running by executing 'eval `ssh-agent`'.")
-
-    # Check if ssh-add -L is not empty
-    ssh_add_output = subprocess.run(["ssh-add", "-L"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    if ssh_add_output.returncode != 0:
-        exit_error("No SSH identities found in the SSH agent. Please run `ssh-add`.")
+    _sanity_reserve()
 
     # check for token
     if not settings.API_TOKEN:
@@ -1340,7 +1698,7 @@ def reserve(
     # test details
     test = TestTMT
     test["url"] = RESERVE_URL
-    test["ref"] = RESERVE_REF
+    test["ref"] = git_ref or RESERVE_REF
     test["name"] = RESERVE_PLAN
 
     # environment details
@@ -1396,41 +1754,37 @@ def reserve(
     if post_install_script:
         environment["settings"]["provisioning"]["post_install_script"] = post_install_script
 
+    # Setting up retries
+    session = requests.Session()
+    install_http_retries(session)
+
     if not skip_workstation_access or security_group_rule_ingress or security_group_rule_egress:
         ingress_rules = security_group_rule_ingress or []
         if not skip_workstation_access:
-            try:
-                get_ip = requests.get(settings.PUBLIC_IP_CHECKER_URL)
-            except requests.exceptions.RequestException as err:
-                exit_error(f"Could not get workstation ip to form a security group rule: {err}")
-
-            if get_ip.ok:
-                ip = get_ip.text.strip()
-                ingress_rules.append(f'-1:{ip}:-1')  # noqa: E231
-
-            else:
-                exit_error(f"Got {get_ip.status_code} while checking {settings.PUBLIC_IP_CHECKER_URL}")
+            ingress_rules.append(_localhost_ingress_rule(session))
 
         rules = _parse_security_group_rules(ingress_rules, security_group_rule_egress or [])
         environment["settings"]["provisioning"].update(rules)
 
     console.print(f"🕗 Reserved for [blue]{str(reservation_duration)}[/blue] minutes")
-    environment["variables"] = {"TF_RESERVATION_DURATION": str(reservation_duration)}
+
+    if "variables" not in environment or environment["variables"] is None:
+        environment["variables"] = {}
+
+    environment["variables"]["TF_RESERVATION_DURATION"] = str(reservation_duration)
+
+    if debug_reservation:
+        environment["variables"]["TF_RESERVATION_DEBUG"] = "1"
 
     authorized_keys = read_glob_paths(ssh_public_keys).encode("utf-8")
     if not authorized_keys:
         exit_error(f"No public SSH keys found under {', '.join(ssh_public_keys)}, cannot continue.")
 
-    # check for ssh-agent, we require it for a pleasant usage
-    if not os.getenv('SSH_AUTH_SOCK'):
-        exit_error("No 'ssh-agent' seems to be running, it is required for reservations to work, cannot continue.")
-
     authorized_keys_bytes = base64.b64encode(authorized_keys)
     environment["secrets"] = {"TF_RESERVATION_AUTHORIZED_KEYS_BASE64": authorized_keys_bytes.decode("utf-8")}
 
     # create final request
     request = TestingFarmRequestV1
-    request["api_key"] = settings.API_TOKEN
     request["test"]["fmf"] = test
 
     # worker image
@@ -1451,10 +1805,6 @@ def reserve(
     # submit request to Testing Farm
     post_url = urllib.parse.urljoin(str(settings.API_URL), "v0.1/requests")
 
-    # Setting up retries
-    session = requests.Session()
-    install_http_retries(session)
-
     # dry run
     if dry_run:
         if print_only_request_id:
@@ -1465,7 +1815,7 @@ def reserve(
         raise typer.Exit()
 
     # handle errors
-    response = session.post(post_url, json=request)
+    response = session.post(post_url, json=request, headers=_get_headers(settings.API_TOKEN))
     if response.status_code == 401:
         exit_error(f"API token is invalid. See {settings.ONBOARDING_DOCS} for more information.")
 
@@ -1524,7 +1874,11 @@ def reserve(
             current_state = state
 
             if state in ["complete", "error"]:
-                exit_error("Reservation failed, check API request or contact Testing Farm")
+                exit_error("Reservation failed, check the API request or contact Testing Farm.")
+
+            if state in ["canceled", "cancel-requested"]:
+                progress.stop()
+                exit_error("Reservation canceled.")
 
             if not print_only_request_id and task_id is not None:
                 progress.update(task_id, description=f"Reservation job is [yellow]{current_state}[/yellow]")
@@ -1579,6 +1933,10 @@ def reserve(
                     )
                 )
 
+            if '[testing-farm-request] Cancelling pipeline' in pipeline_log:
+                progress.stop()
+                exit_error('Pipeline was canceled.')
+
             if '[pre-artifact-installation]' in pipeline_log:
                 current_state = "preparing environment"
 
@@ -1644,7 +2002,7 @@ def cancel(
     install_http_retries(session)
 
     # Get the request details
-    response = session.delete(request_url, json={"api_key": api_token})
+    response = session.delete(request_url, headers=_get_headers(api_token))
 
     if response.status_code == 401:
         exit_error(f"API token is invalid. See {settings.ONBOARDING_DOCS} for more information.")
@@ -1662,3 +2020,74 @@ def cancel(
         exit_error(f"Unexpected error. Please file an issue to {settings.ISSUE_TRACKER}.")
 
     console.print("✅ Request [yellow]cancellation requested[/yellow]. It will be canceled soon.")
+
+
+def encrypt(
+    message: str = typer.Argument(..., help="Message to be encrypted."),
+    api_url: str = ARGUMENT_API_URL,
+    api_token: str = ARGUMENT_API_TOKEN,
+    git_url: Optional[str] = typer.Option(
+        None,
+        help="URL of a GIT repository to which the secret will be tied. If not set, it is detected from the current "
+        "git repository.",
+    ),
+    token_id: Optional[str] = typer.Option(
+        None,
+        help="Token ID to which the secret will be tied. If not set, Token ID will be detected from provided Token.",
+    ),
+):
+    """
+    Create secrets for use in in-repository configuration.
+    """
+
+    # check for token
+    if not api_token:
+        exit_error("No API token found, export `TESTING_FARM_API_TOKEN` environment variable")
+
+    git_available = bool(shutil.which("git"))
+
+    # resolve git repository details from the current repository
+    if not git_url:
+        if not git_available:
+            exit_error("no git url defined")
+        git_url = cmd_output_or_exit("git remote get-url origin", "could not auto-detect git url")
+        # use https instead git when auto-detected
+        # GitLab: git@github.com:containers/podman.git
+        # GitHub: git@gitlab.com:testing-farm/cli.git, git+ssh://git@gitlab.com/spoore/centos_rpms_jq.git
+        # Pagure: ssh://git@pagure.io/fedora-ci/messages.git
+        assert git_url
+        git_url = re.sub(r"^(?:(?:git\+)?ssh://)?git@([^:/]*)[:/](.*)", r"https://\1/\2", git_url)
+
+    payload = {'url': git_url, 'message': message}
+
+    if token_id:
+        payload['token_id'] = token_id
+        console_stderr.print(f'🔒 Encrypting secret for token id {token_id} for repository {git_url}')
+    else:
+        console_stderr.print(f'🔒 Encrypting secret for your token in repo {git_url}')
+
+    # submit request to Testing Farm
+    post_url = urllib.parse.urljoin(api_url, "/v0.1/secrets/encrypt")
+
+    session = requests.Session()
+    response = session.post(post_url, json=payload, headers={'Authorization': f'Bearer {api_token}'})
+
+    # handle errors
+    if response.status_code == 401:
+        exit_error(f"API token is invalid. See {settings.ONBOARDING_DOCS} for more information.")
+
+    if response.status_code == 400:
+        exit_error(
+            f"Request is invalid. {response.json().get('message') or 'Reason unknown.'}."
+            f"\nPlease file an issue to {settings.ISSUE_TRACKER} if unsure."
+        )
+
+    if response.status_code != 200:
+        console_stderr.print(response.text)
+        exit_error(f"Unexpected error. Please file an issue to {settings.ISSUE_TRACKER}.")
+
+    console_stderr.print(
+        "💡 See https://docs.testing-farm.io/Testing%20Farm/0.1/test-request.html#secrets-in-repo-config for more "
+        "information on how to store the secret in repository."
+    )
+    console.print(response.text)