paasta-tools 1.27.0__py3-none-any.whl → 1.35.8__py3-none-any.whl

paasta_tools/cli/cmds/mark_for_deployment.py

@@ -157,6 +157,8 @@ def add_subparser(subparsers: argparse._SubParsersAction) -> None:
         help="Name of the service which you wish to mark for deployment. Leading "
         '"services-" will be stripped.',
         required=True,
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     )
     arg_service.completer = lazy_choices_completer(list_services)  # type: ignore
     list_parser.add_argument(
@@ -1157,7 +1159,7 @@ class MarkForDeploymentProcess(RollbackSlackDeploymentProcess):
 
     def on_enter_deploy_errored(self) -> None:
         report_waiting_aborted(self.service, self.deploy_group)
-        self.update_slack_status(f"Deploy aborted, but it will still try to converge.")
+        self.update_slack_status("Deploy aborted, but it will still try to converge.")
         self.send_manual_rollback_instructions()
         if self.deploy_group_is_set_to_notify("notify_after_abort"):
             self.ping_authors("Deploy errored")
@@ -1264,7 +1266,15 @@ class MarkForDeploymentProcess(RollbackSlackDeploymentProcess):
             self.ping_authors()
 
     def send_manual_rollback_instructions(self) -> None:
-        if self.deployment_version != self.old_deployment_version:
+        # NOTE: new deploy groups are not exactly a particularly frequent occurrence, but
+        # we want to prevent sending messages that look like
+        # `If you need to roll back manually, run: paasta rollback --service $S --deploy-group $G --commit None`
+        # since that's not actually valid/actionable.
+        # thus the seemingly out-of-nowhere old_git_sha check: new deploy groups won't have value set there.
+        if (
+            self.deployment_version != self.old_deployment_version
+            and self.old_git_sha is not None
+        ):
             extra_rollback_args = ""
             if self.old_deployment_version.image_version:
                 extra_rollback_args = (
@@ -1317,7 +1327,7 @@ class MarkForDeploymentProcess(RollbackSlackDeploymentProcess):
         inactive_button_texts = {
             "forward": f"Continue Forward to {version_short_str} :arrow_forward:",
             "complete": f"Complete deploy to {version_short_str} :white_check_mark:",
-            "snooze": f"Reset countdown",
+            "snooze": "Reset countdown",
             "enable_auto_rollbacks": "Enable auto rollbacks :eyes:",
             "disable_auto_rollbacks": "Disable auto rollbacks :close_eyes_monkey:",
         }
@@ -1843,7 +1853,7 @@ async def wait_for_deployment(
             system_paasta_config.get_mark_for_deployment_default_time_before_first_diagnosis()
         )
 
-    with progressbar.ProgressBar(maxval=total_instances) as bar:
+    with progressbar.ProgressBar(max_value=total_instances) as bar:
         instance_done_futures = []
         with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
             for cluster, instance_configs in instance_configs_per_cluster.items():

paasta_tools/cli/cmds/mesh_status.py

@@ -44,9 +44,10 @@ def add_subparser(subparsers) -> None:
     mesh_status_parser.add_argument(
         "-s",
         "--service",
-        type=str,
         help="The name of the service you wish to inspect",
         required=True,
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     mesh_status_parser.add_argument(
         "-i",
@@ -124,7 +125,7 @@ def paasta_mesh_status_on_api_endpoint(
             [PaastaColors.red(f"Could not connect to API: {exc.__class__.__name__}")],
         )
     except Exception as e:
-        output = [PaastaColors.red(f"Exception when talking to the API:")]
+        output = [PaastaColors.red("Exception when talking to the API:")]
         output.extend(str(e).split("\n"))
         return 1, output
 

paasta_tools/cli/cmds/push_to_registry.py

@@ -62,6 +62,8 @@ def add_subparser(subparsers: argparse._SubParsersAction) -> None:
         help='Name of service for which you wish to upload a docker image. Leading "services-", '
         "as included in a Jenkins job name, will be stripped.",
         required=True,
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     )
     list_parser.add_argument(
         "-c",

paasta_tools/cli/cmds/remote_run.py

@@ -14,6 +14,7 @@
 # limitations under the License.
 import argparse
 import json
+import os
 import shutil
 import subprocess
 import sys
@@ -24,6 +25,8 @@ from paasta_tools.cli.utils import get_paasta_oapi_api_clustername
 from paasta_tools.cli.utils import get_paasta_oapi_client_with_auth
 from paasta_tools.cli.utils import lazy_choices_completer
 from paasta_tools.cli.utils import run_interactive_cli
+from paasta_tools.kubernetes.remote_run import format_remote_run_job_name
+from paasta_tools.kubernetes.remote_run import load_eks_or_adhoc_deployment_config
 from paasta_tools.kubernetes.remote_run import TOOLBOX_MOCK_SERVICE
 from paasta_tools.paastaapi.exceptions import ApiException
 from paasta_tools.paastaapi.model.remote_run_start import RemoteRunStart
@@ -39,8 +42,14 @@ from paasta_tools.utils import SystemPaastaConfig
 KUBECTL_EXEC_CMD_TEMPLATE = (
     "{kubectl_wrapper} --token {token} exec -it -n {namespace} {pod} -- /bin/bash"
 )
-KUBECTL_CP_CMD_TEMPLATE = (
-    "{kubectl_wrapper} --token {token} -n {namespace} cp {filename} {pod}:/tmp/"
+KUBECTL_CP_TO_CMD_TEMPLATE = (
+    "{kubectl_wrapper} --token {token} -n {namespace} cp {source} {pod}:{dest}"
+)
+KUBECTL_CP_FROM_CMD_TEMPLATE = (
+    "{kubectl_wrapper} --token {token} -n {namespace} cp {pod}:{source} {dest}"
+)
+KUBECTL_LOGS_CMD_TEMPLATE = (
+    "{kubectl_wrapper} --token {token} logs -n {namespace} -f {pod}"
 )
 
 
@@ -59,6 +68,13 @@ def _list_services_and_toolboxes() -> List[str]:
     )
 
 
+def _get_kubectl_wrapper(cluster: str) -> str:
+    kubectl_wrapper = f"kubectl-eks-{cluster}"
+    if not shutil.which(kubectl_wrapper):
+        kubectl_wrapper = f"kubectl-{cluster}"
+    return kubectl_wrapper
+
+
 def parse_error(body: str) -> str:
     try:
         body_object = json.loads(body)
@@ -71,6 +87,81 @@ def parse_error(body: str) -> str:
     )
 
 
+def paasta_remote_run_copy(
+    args: argparse.Namespace,
+    system_paasta_config: SystemPaastaConfig,
+) -> int:
+    client = get_paasta_oapi_client_with_auth(
+        cluster=get_paasta_oapi_api_clustername(cluster=args.cluster, is_eks=True),
+        system_paasta_config=system_paasta_config,
+    )
+    if not client:
+        print("Cannot get a paasta-api client")
+        return 1
+
+    # Create the config and extract the job name
+    user = get_username()
+    deployment_config = load_eks_or_adhoc_deployment_config(
+        args.service, args.instance, args.cluster, args.toolbox, user
+    )
+    deployment_name = deployment_config.get_sanitised_deployment_name()
+    job_name = format_remote_run_job_name(deployment_name, user)
+
+    poll_response = client.remote_run.remote_run_poll(
+        service=args.service,
+        instance=args.instance,
+        job_name=job_name,
+        user=user,
+        toolbox=args.toolbox,
+    )
+    if poll_response.status != 200:
+        print(
+            "Unable to find running remote-run pod: have you started one with `paasta remote-run start`?"
+        )
+        return 1
+
+    if args.to_pod:
+        # Create the toolbox copy command
+        exec_command = f"scp -A {args.copy_file_source} {poll_response.pod_address}:{args.copy_file_dest}"
+        # Pick the correct template and force /tmp/ for non-toolbox
+        kubectl_cp_template = KUBECTL_CP_TO_CMD_TEMPLATE
+        if not args.copy_file_dest.startswith("/tmp") and not args.toolbox:
+            args.copy_file_dest = os.path.join("/tmp/", args.copy_file_dest)
+    else:
+        exec_command = f"scp -A {poll_response.pod_address}:{args.copy_file_source} {args.copy_file_dest}"
+        kubectl_cp_template = KUBECTL_CP_FROM_CMD_TEMPLATE
+
+    # Kubectl cp doesnt like directories as a destination
+    if os.path.isdir(args.copy_file_dest):
+        args.copy_file_dest = os.path.join(
+            args.copy_file_dest, os.path.basename(args.copy_file_source)
+        )
+
+    if args.toolbox:
+        cp_command = exec_command.split(" ")
+    else:
+        token_response = client.remote_run.remote_run_token(
+            args.service, args.instance, user
+        )
+        kubectl_wrapper = _get_kubectl_wrapper(args.cluster)
+        cp_command = kubectl_cp_template.format(
+            kubectl_wrapper=kubectl_wrapper,
+            namespace=poll_response.namespace,
+            pod=poll_response.pod_name,
+            source=args.copy_file_source,
+            dest=args.copy_file_dest,
+            token=token_response.token,
+        ).split(" ")
+    call = subprocess.run(cp_command, capture_output=True)
+    if call.returncode != 0:
+        print("Error copying file from remote-run pod: ", file=sys.stderr)
+        print(call.stderr.decode("utf-8"), file=sys.stderr)
+        return 1
+    print(f"{args.copy_file_source} successfully copied to {args.copy_file_dest}")
+
+    return 0
+
+
 def paasta_remote_run_start(
     args: argparse.Namespace,
     system_paasta_config: SystemPaastaConfig,
@@ -84,7 +175,6 @@ def paasta_remote_run_start(
     if not client:
         print("Cannot get a paasta-api client")
         return 1
-
     user = get_username()
     try:
         start_response = client.remote_run.remote_run_start(
@@ -96,6 +186,7 @@ def paasta_remote_run_start(
                 recreate=args.recreate,
                 max_duration=args.max_duration,
                 toolbox=args.toolbox,
+                command=args.cmd or "",
             ),
         )
     except ApiException as e:
@@ -132,7 +223,7 @@ def paasta_remote_run_start(
         print(f"{status_prefix}Timed out while waiting for job to start")
         return 1
 
-    if not args.interactive and not args.toolbox:
+    if not (args.interactive or args.toolbox or args.follow):
         print("Successfully started remote-run job")
         return 0
 
@@ -148,10 +239,10 @@ def paasta_remote_run_start(
         token_response = client.remote_run.remote_run_token(
             args.service, args.instance, user
         )
-        kubectl_wrapper = f"kubectl-eks-{args.cluster}"
-        if not shutil.which(kubectl_wrapper):
-            kubectl_wrapper = f"kubectl-{args.cluster}"
-        exec_command = KUBECTL_EXEC_CMD_TEMPLATE.format(
+        kubectl_wrapper = _get_kubectl_wrapper(args.cluster)
+        exec_command = (
+            KUBECTL_LOGS_CMD_TEMPLATE if args.follow else KUBECTL_EXEC_CMD_TEMPLATE
+        ).format(
             kubectl_wrapper=kubectl_wrapper,
             namespace=poll_response.namespace,
             pod=poll_response.pod_name,
@@ -160,11 +251,12 @@ def paasta_remote_run_start(
 
     if args.copy_file:
         for filename in args.copy_file:
-            cp_command = KUBECTL_CP_CMD_TEMPLATE.format(
+            cp_command = KUBECTL_CP_TO_CMD_TEMPLATE.format(
                 kubectl_wrapper=kubectl_wrapper,
                 namespace=poll_response.namespace,
                 pod=poll_response.pod_name,
-                filename=filename,
+                source=filename,
+                dest=os.path.join("/tmp", os.path.basename(filename)),
                 token=token_response.token,
             ).split(" ")
             call = subprocess.run(cp_command, capture_output=True)
@@ -203,11 +295,20 @@ def paasta_remote_run_stop(
 
 
 def add_common_args_to_parser(parser: argparse.ArgumentParser):
+    def _validate_service_name(name: str) -> str:
+        # remove trailing slash if present for folks tab-completing directories
+        if name.rstrip("/") not in _list_services_and_toolboxes():
+            raise ValueError(f"{name} is not a known service name")
+        return name
+
     service_arg = parser.add_argument(
         "-s",
         "--service",
         help="The name of the service you wish to inspect. Required.",
         required=True,
+        # not using `choices` for validation to avoid the help text
+        # for the command being incredibly large
+        type=_validate_service_name,
     )
     service_arg.completer = lazy_choices_completer(_list_services_and_toolboxes)  # type: ignore
     instance_or_toolbox = parser.add_mutually_exclusive_group()
@@ -231,6 +332,7 @@ def add_common_args_to_parser(parser: argparse.ArgumentParser):
         "--cluster",
         help="The name of the cluster you wish to run your task on. Required.",
         required=True,
+        choices=list_clusters(),
     )
     cluster_arg.completer = lazy_choices_completer(list_clusters)  # type: ignore
 
@@ -249,7 +351,8 @@ def add_subparser(subparsers: argparse._SubParsersAction) -> None:
         description="Starts or connects to a remote-run-job",
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
-    start_parser.add_argument(
+    cmd_or_interactive = start_parser.add_mutually_exclusive_group()
+    cmd_or_interactive.add_argument(
         "-I",
         "--interactive",
         help=(
@@ -259,6 +362,15 @@ def add_subparser(subparsers: argparse._SubParsersAction) -> None:
         action="store_true",
         default=False,
     )
+    cmd_or_interactive.add_argument(
+        "-C",
+        "--cmd",
+        help=(
+            "Run container with particular command, rather than the one specified in soa-configs"
+        ),
+        required=False,
+        default=None,
+    )
     start_parser.add_argument(
         "-m",
         "--max-duration",
@@ -267,7 +379,14 @@ def add_subparser(subparsers: argparse._SubParsersAction) -> None:
             "automatically stopped (capped by the API backend)"
         ),
         type=int,
-        default=1800,
+        default=7200,
+    )
+    start_parser.add_argument(
+        "-f",
+        "--follow",
+        help="Attach to and follow container output",
+        action="store_true",
+        default=False,
     )
     start_parser.add_argument(
         "-r",
@@ -295,8 +414,31 @@ def add_subparser(subparsers: argparse._SubParsersAction) -> None:
         description="Stop your remote-run job if it exists",
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
+    copy_parser = subparsers.add_parser(
+        "copy",
+        help="Copy a file from a running remote-run job",
+        description="Copy a file from a running remote-run job",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    copy_parser.add_argument(
+        "copy_file_source",
+        help="Location of the file within the pod",
+        type=str,
+    )
+    copy_parser.add_argument(
+        "copy_file_dest",
+        help="Location on the local host to copy the file to",
+        type=str,
+    )
+    copy_parser.add_argument(
+        "--to-pod",
+        help="Copy a file to the pod instead of from the pod",
+        action="store_true",
+        default=False,
+    )
     add_common_args_to_parser(start_parser)
     add_common_args_to_parser(stop_parser)
+    add_common_args_to_parser(copy_parser)
     remote_run_parser.set_defaults(command=paasta_remote_run)
 
 
@@ -306,4 +448,6 @@ def paasta_remote_run(args: argparse.Namespace) -> int:
         return paasta_remote_run_start(args, system_paasta_config)
     elif args.remote_run_command == "stop":
         return paasta_remote_run_stop(args, system_paasta_config)
+    elif args.remote_run_command == "copy":
+        return paasta_remote_run_copy(args, system_paasta_config)
     raise ValueError(f"Unsupported subcommand: {args.remote_run_command}")

paasta_tools/cli/cmds/rollback.py

@@ -107,7 +107,11 @@ def add_subparser(subparsers: argparse._SubParsersAction) -> None:
     )
 
     arg_service = list_parser.add_argument(
-        "-s", "--service", help='Name of the service to rollback (e.g. "service1")'
+        "-s",
+        "--service",
+        help='Name of the service to rollback (e.g. "service1")',
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     )
     arg_service.completer = lazy_choices_completer(list_services)  # type: ignore
     list_parser.add_argument(
@@ -340,7 +344,7 @@ def paasta_rollback(args: argparse.Namespace) -> int:
         )
         print(
             PaastaColors.yellow(
-                f"WARNING: Failing to do so means that Jenkins will redeploy the latest code on the next scheduled build!"
+                "WARNING: Failing to do so means that Jenkins will redeploy the latest code on the next scheduled build!"
             )
         )
 

paasta_tools/cli/cmds/secret.py

@@ -212,7 +212,7 @@ def _add_vault_auth_args(parser: argparse.ArgumentParser):
         dest="vault_auth_method",
         required=False,
         default="token",
-        choices=["token", "ldap"],
+        choices=["token", "okta"],
     )
     parser.add_argument(
         "--vault-token-file",
@@ -246,6 +246,8 @@ def _add_common_args(parser: argparse.ArgumentParser, allow_shared: bool = True)
         "--service",
         required=not allow_shared,
         help="The name of the service on which you wish to act",
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
 
     if allow_shared:
@@ -456,7 +458,7 @@ def paasta_secret(args):
                 "vault_token_file": args.vault_token_file,
                 # best solution so far is to change the below string to "token",
                 # so that token file is picked up from argparse
-                "vault_auth_method": "ldap",  # must have LDAP to get 2FA push for prod
+                "vault_auth_method": "okta",  # must use Okta to get 2FA push
             },
         )
         secret_provider.write_secret(

paasta_tools/cli/cmds/security_check.py

@@ -28,6 +28,8 @@ def add_subparser(subparsers):
         help='Name of service for which you wish to check. Leading "services-", as included in a '
         "Jenkins job name, will be stripped.",
         required=True,
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     )
     list_parser.add_argument(
         "-c", "--commit", help="Git sha of the image to check", required=True

paasta_tools/cli/cmds/spark_run.py

@@ -238,6 +238,8 @@ def add_subparser(subparsers):
         "--service",
         help="The name of the service from which the Spark image is built.",
         default=DEFAULT_SPARK_SERVICE,
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
 
     list_parser.add_argument(
@@ -897,20 +899,22 @@ def configure_and_run_docker_container(
         )
     )  # type:ignore
     environment.update(extra_driver_envs)
+    if "jupyter-lab" == args.cmd:
+        environment["SPARK_DRIVER_TYPE"] = "jupyter"
 
     if args.use_service_auth_token:
         environment["YELP_SVC_AUTHZ_TOKEN"] = get_service_auth_token()
 
     webui_url = get_webui_url(spark_conf["spark.ui.port"])
-    webui_url_msg = PaastaColors.green(f"\nSpark monitoring URL: ") + f"{webui_url}\n"
+    webui_url_msg = PaastaColors.green("\nSpark monitoring URL: ") + f"{webui_url}\n"
 
     docker_cmd = get_docker_cmd(args, instance_config, spark_conf_str)
     if "history-server" in docker_cmd:
-        print(PaastaColors.green(f"\nSpark history server URL: ") + f"{webui_url}\n")
+        print(PaastaColors.green("\nSpark history server URL: ") + f"{webui_url}\n")
     elif any(c in docker_cmd for c in ["pyspark", "spark-shell", "spark-submit"]):
         grafana_url = get_grafana_url(spark_conf)
         dashboard_url_msg = (
-            PaastaColors.green(f"\nGrafana dashboard: ") + f"{grafana_url}\n"
+            PaastaColors.green("\nGrafana dashboard: ") + f"{grafana_url}\n"
         )
         print(webui_url_msg)
         print(dashboard_url_msg)