paasta-tools 1.21.3__py3-none-any.whl

paasta_tools/cli/cmds/remote_run.py

@@ -0,0 +1,252 @@
+#!/usr/bin/env python
+# Copyright 2015-2017 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import shutil
+import time
+from typing import List
+
+from paasta_tools.cli.utils import get_paasta_oapi_api_clustername
+from paasta_tools.cli.utils import get_paasta_oapi_client_with_auth
+from paasta_tools.cli.utils import lazy_choices_completer
+from paasta_tools.cli.utils import run_interactive_cli
+from paasta_tools.kubernetes.remote_run import TOOLBOX_MOCK_SERVICE
+from paasta_tools.paastaapi.model.remote_run_start import RemoteRunStart
+from paasta_tools.paastaapi.model.remote_run_stop import RemoteRunStop
+from paasta_tools.utils import get_username
+from paasta_tools.utils import list_all_instances_for_service
+from paasta_tools.utils import list_clusters
+from paasta_tools.utils import list_services
+from paasta_tools.utils import load_system_paasta_config
+from paasta_tools.utils import SystemPaastaConfig
+
+
+KUBECTL_CMD_TEMPLATE = (
+    "{kubectl_wrapper} --token {token} exec -it -n {namespace} {pod} -- /bin/bash"
+)
+
+
+def _list_services_and_toolboxes() -> List[str]:
+    try:
+        toolbox_instances = list_all_instances_for_service(
+            TOOLBOX_MOCK_SERVICE, instance_type="adhoc"
+        )
+    except Exception:
+        toolbox_instances = set()
+    # NOTE: API authorization is enforced by service, and we want different rules
+    # for each toolbox, so we combine service and instance in this case to properly
+    # allow that to happen.
+    return list(list_services()) + sorted(
+        f"{TOOLBOX_MOCK_SERVICE}-{instance}" for instance in toolbox_instances
+    )
+
+
+def paasta_remote_run_start(
+    args: argparse.Namespace,
+    system_paasta_config: SystemPaastaConfig,
+) -> int:
+    client = get_paasta_oapi_client_with_auth(
+        cluster=get_paasta_oapi_api_clustername(cluster=args.cluster, is_eks=True),
+        system_paasta_config=system_paasta_config,
+    )
+    if not client:
+        print("Cannot get a paasta-api client")
+        return 1
+
+    user = get_username()
+    start_response = client.remote_run.remote_run_start(
+        args.service,
+        args.instance,
+        RemoteRunStart(
+            user=user,
+            interactive=args.interactive,
+            recreate=args.recreate,
+            max_duration=args.max_duration,
+            toolbox=args.toolbox,
+        ),
+    )
+    if start_response.status >= 300:
+        print(f"Error from PaaSTA APIs while starting job: {start_response.message}")
+        return 1
+
+    print(
+        f"Triggered remote-run job for {args.service}. Waiting for pod to come online..."
+    )
+    start_time = time.time()
+    while time.time() - start_time < args.timeout:
+        poll_response = client.remote_run.remote_run_poll(
+            service=args.service,
+            instance=args.instance,
+            job_name=start_response.job_name,
+            user=user,
+            toolbox=args.toolbox,
+        )
+        if poll_response.status == 200:
+            print("")
+            break
+        print(f"\rStatus: {poll_response.message}", end="")
+        time.sleep(10)
+    else:
+        print("Timed out while waiting for job to start")
+        return 1
+
+    if not args.interactive and not args.toolbox:
+        print("Successfully started remote-run job")
+        return 0
+
+    print("Pod ready, establishing interactive session...")
+
+    if args.toolbox:
+        # NOTE: we only do this for toolbox containers since those images are built with interactive
+        # access in mind, and SSH sessions provide better auditability of user actions.
+        # I.e., being `nobody` is fine in a normal remote-run, but in toolbox containers
+        # we will require knowing the real user (and some tools may need that too).
+        exec_command = f"ssh -A {poll_response.pod_address}"
+    else:
+        token_response = client.remote_run.remote_run_token(
+            args.service, args.instance, user
+        )
+        kubectl_wrapper = f"kubectl-eks-{args.cluster}"
+        if not shutil.which(kubectl_wrapper):
+            kubectl_wrapper = f"kubectl-{args.cluster}"
+        exec_command = KUBECTL_CMD_TEMPLATE.format(
+            kubectl_wrapper=kubectl_wrapper,
+            namespace=poll_response.namespace,
+            pod=poll_response.pod_name,
+            token=token_response.token,
+        )
+
+    run_interactive_cli(exec_command)
+    return 0
+
+
+def paasta_remote_run_stop(
+    args: argparse.Namespace,
+    system_paasta_config: SystemPaastaConfig,
+) -> int:
+    client = get_paasta_oapi_client_with_auth(
+        cluster=get_paasta_oapi_api_clustername(cluster=args.cluster, is_eks=True),
+        system_paasta_config=system_paasta_config,
+    )
+    if not client:
+        print("Cannot get a paasta-api client")
+        return 1
+    response = client.remote_run.remote_run_stop(
+        args.service,
+        args.instance,
+        RemoteRunStop(user=get_username(), toolbox=args.toolbox),
+    )
+    print(response.message)
+    return 0 if response.status < 300 else 1
+
+
+def add_common_args_to_parser(parser: argparse.ArgumentParser):
+    service_arg = parser.add_argument(
+        "-s",
+        "--service",
+        help="The name of the service you wish to inspect. Required.",
+        required=True,
+    )
+    service_arg.completer = lazy_choices_completer(_list_services_and_toolboxes)  # type: ignore
+    instance_or_toolbox = parser.add_mutually_exclusive_group()
+    instance_or_toolbox.add_argument(
+        "-i",
+        "--instance",
+        help=(
+            "Simulate a docker run for a particular instance of the "
+            "service, like 'main' or 'canary'. Required."
+        ),
+        default="main",
+    )
+    instance_or_toolbox.add_argument(
+        "--toolbox",
+        help="The selected service is a 'toolbox' container",
+        action="store_true",
+        default=False,
+    )
+    cluster_arg = parser.add_argument(
+        "-c",
+        "--cluster",
+        help="The name of the cluster you wish to run your task on. Required.",
+        required=True,
+    )
+    cluster_arg.completer = lazy_choices_completer(list_clusters)  # type: ignore
+
+
+def add_subparser(subparsers: argparse._SubParsersAction) -> None:
+    remote_run_parser = subparsers.add_parser(
+        "remote-run",
+        help="Run services / jobs remotely",
+        description="'paasta remote-run' runs services / jobs remotely",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    subparsers = remote_run_parser.add_subparsers(dest="remote_run_command")
+    start_parser = subparsers.add_parser(
+        "start",
+        help="Start or connect to a remote-run job",
+        description="Starts or connects to a remote-run-job",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    start_parser.add_argument(
+        "-I",
+        "--interactive",
+        help=(
+            "Run container in interactive mode. If interactive is set the "
+            'default command will be "bash" unless otherwise set by the "--cmd" flag'
+        ),
+        action="store_true",
+        default=False,
+    )
+    start_parser.add_argument(
+        "-m",
+        "--max-duration",
+        help=(
+            "Amount of time in seconds after which the job is "
+            "automatically stopped (capped by the API backend)"
+        ),
+        type=int,
+        default=1800,
+    )
+    start_parser.add_argument(
+        "-r",
+        "--recreate",
+        help="Recreate remote-run job if already existing",
+        action="store_true",
+        default=False,
+    )
+    start_parser.add_argument(
+        "-t",
+        "--timeout",
+        help="Maximum time to wait for a job to start, in seconds",
+        type=int,
+        default=600,
+    )
+    stop_parser = subparsers.add_parser(
+        "stop",
+        help="Stop your remote-run job if it exists",
+        description="Stop your remote-run job if it exists",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    add_common_args_to_parser(start_parser)
+    add_common_args_to_parser(stop_parser)
+    remote_run_parser.set_defaults(command=paasta_remote_run)
+
+
+def paasta_remote_run(args: argparse.Namespace) -> int:
+    system_paasta_config = load_system_paasta_config()
+    if args.remote_run_command == "start":
+        return paasta_remote_run_start(args, system_paasta_config)
+    elif args.remote_run_command == "stop":
+        return paasta_remote_run_stop(args, system_paasta_config)
+    raise ValueError(f"Unsupported subcommand: {args.remote_run_command}")

paasta_tools/cli/cmds/rollback.py

@@ -0,0 +1,347 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+from typing import Collection
+from typing import Dict
+from typing import Generator
+from typing import Mapping
+from typing import Tuple
+
+from humanize import naturaltime
+
+from paasta_tools.cli.cmds.mark_for_deployment import can_user_deploy_service
+from paasta_tools.cli.cmds.mark_for_deployment import get_deploy_info
+from paasta_tools.cli.cmds.mark_for_deployment import mark_for_deployment
+from paasta_tools.cli.utils import extract_tags
+from paasta_tools.cli.utils import figure_out_service_name
+from paasta_tools.cli.utils import lazy_choices_completer
+from paasta_tools.cli.utils import list_deploy_groups
+from paasta_tools.cli.utils import validate_full_git_sha
+from paasta_tools.cli.utils import validate_given_deploy_groups
+from paasta_tools.deployment_utils import get_currently_deployed_version
+from paasta_tools.remote_git import list_remote_refs
+from paasta_tools.utils import _log_audit
+from paasta_tools.utils import datetime_from_utc_to_local
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import DeploymentVersion
+from paasta_tools.utils import format_table
+from paasta_tools.utils import get_git_url
+from paasta_tools.utils import list_services
+from paasta_tools.utils import PaastaColors
+from paasta_tools.utils import parse_timestamp
+from paasta_tools.utils import RollbackTypes
+
+
+def add_subparser(subparsers: argparse._SubParsersAction) -> None:
+    list_parser = subparsers.add_parser(
+        "rollback",
+        help="Rollback a docker image to a previous deploy",
+        description=(
+            "'paasta rollback' is a human-friendly tool for marking a particular "
+            "docker image for deployment, which invokes a bounce. While the command "
+            "is called 'rollback', it can be used to roll forward or back, as long "
+            "as there is a docker image available for the input Git SHA."
+        ),
+        epilog=(
+            "This rollback command uses the Git control plane, which requires network "
+            "connectivity as well as authorization to the Git repo.\n\n"
+            + PaastaColors.yellow(
+                "WARNING: You MUST manually revert changes in Git and go through the normal push process after using this command.\n"
+            )
+            + PaastaColors.yellow(
+                "WARNING: Failing to do so means that Jenkins will redeploy the latest code on the next scheduled build!"
+            )
+        ),
+        # we manually format the epilog to add newlines + give it an attention-grabbing color
+        # re: reverting changes in Git post-rollback
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    arg_commit = list_parser.add_argument(
+        "-k",
+        "--commit",
+        help="Git SHA to mark for rollback. "
+        "A commit to rollback to is required for paasta rollback to run. However if one is not provided, "
+        "paasta rollback will instead output a list of valid git shas to rollback to.",
+        required=False,
+    )
+    arg_commit.completer = lazy_choices_completer(list_previously_deployed_shas)  # type: ignore
+    arg_version = list_parser.add_argument(
+        "-i",
+        "--image-version",
+        help="Extra version metadata to mark for rollback. "
+        "If your service has enabled no-commit redeploys, both a commit and the extra metadata is required for paasta rollback to run. However if one is not provided, "
+        "paasta rollback will instead output a list of valid versions to rollback to.",
+        required=False,
+        default=None,
+    )
+    arg_version.completer = lazy_choices_completer(list_previously_deployed_image_versions)  # type: ignore
+    arg_deploy_group = list_parser.add_argument(
+        "-l",
+        "--deploy-groups",
+        help="Mark one or more deploy groups to roll back (e.g. "
+        '"all.main", "all.main,all.canary"). If no deploy groups specified,'
+        "no deploy groups for that service are rolled back. To rollback all deploy groups "
+        "use the flag -a or --all-deploy-groups",
+        default="",
+        required=False,
+    )
+    arg_deploy_group.completer = lazy_choices_completer(list_deploy_groups)  # type: ignore
+    list_parser.add_argument(
+        "-a",
+        "--all-deploy-groups",
+        help="Rollback all deploy groups for the service",
+        action="store_true",
+        required=False,
+    )
+
+    arg_service = list_parser.add_argument(
+        "-s", "--service", help='Name of the service to rollback (e.g. "service1")'
+    )
+    arg_service.completer = lazy_choices_completer(list_services)  # type: ignore
+    list_parser.add_argument(
+        "-y",
+        "-d",
+        "--soa-dir",
+        dest="soa_dir",
+        metavar="SOA_DIR",
+        default=DEFAULT_SOA_DIR,
+        help="define a different soa config directory",
+    )
+    list_parser.add_argument(
+        "-f",
+        "--force",
+        help=("Do not check if Git SHA was marked for deployment previously."),
+        action="store_true",
+    )
+    list_parser.set_defaults(command=paasta_rollback)
+
+
+def list_previously_deployed_shas(
+    parsed_args: argparse.Namespace, **kwargs: None
+) -> Generator[str, None, None]:
+    service = parsed_args.service
+    soa_dir = parsed_args.soa_dir
+    deploy_groups = {
+        deploy_group
+        for deploy_group in parsed_args.deploy_groups.split(",")
+        if deploy_group
+    }
+    return (
+        version.sha
+        for version in get_versions_for_service(service, deploy_groups, soa_dir)
+    )
+
+
+def list_previously_deployed_image_versions(
+    parsed_args: argparse.Namespace, **kwargs: None
+) -> Generator[str, None, None]:
+    service = parsed_args.service
+    soa_dir = parsed_args.soa_dir
+    deploy_groups = {
+        deploy_group
+        for deploy_group in parsed_args.deploy_groups.split(",")
+        if deploy_group
+    }
+    return (
+        version.image_version
+        for version in get_versions_for_service(service, deploy_groups, soa_dir)
+    )
+
+
+def get_versions_for_service(
+    service: str, deploy_groups: Collection[str], soa_dir: str
+) -> Mapping[DeploymentVersion, Tuple[str, str]]:
+    """Returns a dictionary of 2-tuples of the form (timestamp, deploy_group) for each version tuple of (deploy sha, image_version)"""
+    if service is None:
+        return {}
+    git_url = get_git_url(service=service, soa_dir=soa_dir)
+    all_deploy_groups = list_deploy_groups(service=service, soa_dir=soa_dir)
+    deploy_groups, _ = validate_given_deploy_groups(all_deploy_groups, deploy_groups)
+    previously_deployed_versions: Dict[DeploymentVersion, Tuple[str, str]] = {}
+    for ref, sha in list_remote_refs(git_url).items():
+        regex_match = extract_tags(ref)
+        try:
+            deploy_group = regex_match["deploy_group"]
+            tstamp = regex_match["tstamp"]
+            image_version = regex_match["image_version"]
+        except KeyError:
+            pass
+        else:
+            # Now we filter and dedup by picking the most recent sha for a deploy group
+            # Note that all strings are greater than ''
+            if deploy_group in deploy_groups:
+                version = DeploymentVersion(sha=sha, image_version=image_version)
+                tstamp_so_far = previously_deployed_versions.get(version, ("all", ""))[
+                    1
+                ]
+                if tstamp > tstamp_so_far:
+                    previously_deployed_versions[version] = (tstamp, deploy_group)
+    return previously_deployed_versions
+
+
+def list_previous_versions(
+    service: str,
+    deploy_groups: Collection[str],
+    any_given_deploy_groups: bool,
+    versions: Mapping[DeploymentVersion, Tuple],
+) -> None:
+    def format_timestamp(tstamp: str) -> str:
+        return naturaltime(datetime_from_utc_to_local(parse_timestamp(tstamp)))
+
+    print("Below is a list of recent commits:")
+    # Latest 10 versions sorted by deployment time
+    list_of_versions = sorted(versions.items(), key=lambda x: x[1], reverse=True)[:10]
+    rows = [("Timestamp -- UTC", "Human time", "deploy_group", "Version")]
+    for version, (timestamp, deploy_group) in list_of_versions:
+        rows.extend(
+            [(timestamp, format_timestamp(timestamp), deploy_group, repr(version))]
+        )
+    for line in format_table(rows):
+        print(line)
+    if len(list_of_versions) >= 2:
+        version, (timestamp, deploy_group) = list_of_versions[1]
+        deploy_groups_arg_line = (
+            "-l %s " % ",".join(deploy_groups) if any_given_deploy_groups else ""
+        )
+        version_arg = (
+            f" --image-version {version.image_version}" if version.image_version else ""
+        )
+        print(
+            "\nFor example, to use the second to last version from {} used on {}, run:".format(
+                format_timestamp(timestamp), PaastaColors.bold(deploy_group)
+            )
+        )
+        print(
+            PaastaColors.bold(
+                f"    paasta rollback -s {service} {deploy_groups_arg_line}-k {version.sha}{version_arg}"
+            )
+        )
+
+
+def paasta_rollback(args: argparse.Namespace) -> int:
+    """Call mark_for_deployment with rollback parameters
+    :param args: contains all the arguments passed onto the script: service,
+    deploy groups and sha. These arguments will be verified and passed onto
+    mark_for_deployment.
+    """
+
+    soa_dir = args.soa_dir
+    service = figure_out_service_name(args, soa_dir)
+
+    deploy_info = get_deploy_info(service=service, soa_dir=args.soa_dir)
+    if not can_user_deploy_service(deploy_info, service):
+        return 1
+
+    git_url = get_git_url(service, soa_dir)
+
+    if args.all_deploy_groups:
+        given_deploy_groups = list_deploy_groups(service=service, soa_dir=soa_dir)
+    else:
+        given_deploy_groups = {
+            deploy_group
+            for deploy_group in args.deploy_groups.split(",")
+            if deploy_group
+        }
+
+    all_deploy_groups = list_deploy_groups(service=service, soa_dir=soa_dir)
+    deploy_groups, invalid = validate_given_deploy_groups(
+        all_deploy_groups, given_deploy_groups
+    )
+
+    if len(invalid) > 0:
+        print(
+            PaastaColors.yellow(
+                "These deploy groups are not valid and will be skipped: %s.\n"
+                % (",").join(invalid)
+            )
+        )
+
+    if len(deploy_groups) == 0 and not args.all_deploy_groups:
+        print(
+            PaastaColors.red(
+                "ERROR: No valid deploy groups specified for %s.\n Use the flag -a to rollback all valid deploy groups for this service"
+                % (service)
+            )
+        )
+        return 1
+
+    versions = get_versions_for_service(service, deploy_groups, soa_dir)
+    commit = args.commit
+    image_version = args.image_version
+    new_version = DeploymentVersion(sha=commit, image_version=image_version)
+    if not commit:
+        print("Please specify a commit to mark for rollback (-k, --commit).")
+        list_previous_versions(
+            service, deploy_groups, bool(given_deploy_groups), versions
+        )
+        return 1
+    elif new_version not in versions and not args.force:
+        print(
+            PaastaColors.red(
+                f"This version {new_version} has never been deployed before."
+            )
+        )
+        print("Please double check it or use --force to skip this verification.\n")
+        list_previous_versions(
+            service, deploy_groups, bool(given_deploy_groups), versions
+        )
+        return 1
+
+    try:
+        validate_full_git_sha(args.commit)
+    except argparse.ArgumentTypeError as e:
+        print(PaastaColors.red(f"Error: {e}"))
+        return 1
+
+    # TODO: Add similar check for when image_version is empty and no-commit redeploys is enforced for requested deploy_group
+
+    returncode = 0
+
+    for deploy_group in deploy_groups:
+        rolled_back_from = get_currently_deployed_version(service, deploy_group)
+        returncode |= mark_for_deployment(
+            git_url=git_url,
+            service=service,
+            deploy_group=deploy_group,
+            commit=commit,
+            image_version=image_version,
+        )
+
+        # we could also gate this by the return code from m-f-d, but we probably care more about someone wanting to
+        # rollback than we care about if the underlying machinery was successfully able to complete the request
+        if rolled_back_from != new_version:
+            audit_action_details = {
+                "rolled_back_from": str(rolled_back_from),
+                "rolled_back_to": str(new_version),
+                "rollback_type": RollbackTypes.USER_INITIATED_ROLLBACK.value,
+                "deploy_group": deploy_group,
+            }
+            _log_audit(
+                action="rollback", action_details=audit_action_details, service=service
+            )
+
+    if returncode == 0:
+        print(
+            PaastaColors.yellow(
+                f"WARNING: You MUST manually revert changes in Git! Use 'git revert {rolled_back_from.sha}', and go through the normal push process. "
+            )
+        )
+        print(
+            PaastaColors.yellow(
+                f"WARNING: Failing to do so means that Jenkins will redeploy the latest code on the next scheduled build!"
+            )
+        )
+
+    return returncode