paasta-tools 1.21.3__py3-none-any.whl

paasta_tools/cli/cmds/get_image_version.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import datetime
+import re
+import sys
+from typing import Optional
+
+from paasta_tools.cli.utils import lazy_choices_completer
+from paasta_tools.cli.utils import PaastaColors
+from paasta_tools.cli.utils import validate_service_name
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import DeploymentsJsonV2Dict
+from paasta_tools.utils import format_timestamp
+from paasta_tools.utils import get_pipeline_deploy_group_configs
+from paasta_tools.utils import list_services
+from paasta_tools.utils import load_v2_deployments_json
+from paasta_tools.utils import optionally_load_system_paasta_config
+from paasta_tools.utils import parse_timestamp
+
+
+def add_subparser(subparsers: argparse._SubParsersAction) -> None:
+    parser = subparsers.add_parser(
+        "get-image-version",
+        help="Returns the value to be used for an image version, which will be used in automated redeploys of the same service SHA. If no deploy groups are configured for automated redeploys, will return no output.",
+    )
+    parser.add_argument(
+        "-f",
+        "--force",
+        help="Force a brand new image_version, regardless if the latest build was recent",
+        action="store_true",
+        default=False,
+    )
+    parser.add_argument(
+        "--max-age",
+        help="max age in seconds (default %(default)s)",
+        type=int,
+        default=2592000,  # TODO: Get from paasta system config
+    )
+    arg_service = parser.add_argument(
+        "-s",
+        "--service",
+        help="Name of the service which you want to get the image version for.",
+        required=True,
+    )
+    arg_service.completer = lazy_choices_completer(list_services)  # type: ignore
+    parser.add_argument(
+        "-c",
+        "--commit",
+        help="Commit to be used with generated image version",
+        required=True,
+    )
+    parser.add_argument(
+        "-y",
+        "--soa-dir",
+        help="A directory from which soa-configs should be read",
+        default=DEFAULT_SOA_DIR,
+    )
+    parser.set_defaults(command=paasta_get_image_version)
+
+
+def check_enable_automated_redeploys(service: str, soa_dir: str) -> bool:
+    deploy_steps = get_pipeline_deploy_group_configs(service, soa_dir)
+    enabled_by_default = (
+        optionally_load_system_paasta_config().get_enable_automated_redeploys_default()
+    )
+    return any(
+        [
+            step.get("enable_automated_redeploys", enabled_by_default)
+            for step in deploy_steps
+        ]
+    )
+
+
+def extract_timestamp(image_version: str) -> Optional[datetime.datetime]:
+    ts_str = re.match(r"^(?P<ts_str>[0-9]{8}T[0-9]{6})", image_version).group()
+    return parse_timestamp(ts_str)
+
+
+def should_generate_new_image_version(old: str, new: str, max_age: int) -> bool:
+    # TODO: Handle additional criteria
+    # Extract dates & compare
+    try:
+        age_diff = (extract_timestamp(new) - extract_timestamp(old)).total_seconds()
+
+        if age_diff < max_age:
+            return False
+        print(
+            PaastaColors.yellow(
+                f"Old image version was {age_diff}s old, generating new one"
+            ),
+            file=sys.stderr,
+        )
+    except Exception as e:
+        print(
+            PaastaColors.red(
+                f"Error: hit an exception {e} checking image version, will create new version {new}"
+            ),
+            file=sys.stderr,
+        )
+    return True
+
+
+def get_latest_image_version(deployments: DeploymentsJsonV2Dict, commit: str) -> str:
+    image_version = None
+    # Image versions start with sortable timestamp
+    # We only care about deployments for this sha; otherwise we will generate a new image_version
+    sorted_image_versions = sorted(
+        [
+            deployment.get("image_version")
+            for deployment in deployments["deployments"].values()
+            if deployment.get("image_version") and deployment.get("git_sha") == commit
+        ],
+        reverse=True,
+    )
+    if sorted_image_versions:
+        image_version = sorted_image_versions[0]
+    return image_version
+
+
+def paasta_get_image_version(args: argparse.Namespace) -> int:
+    service = args.service
+    soa_dir = args.soa_dir
+
+    validate_service_name(service, soa_dir)
+
+    # Check if any deploy groups have set enable_automated_redeploys
+    if not check_enable_automated_redeploys(service, soa_dir):
+        print(
+            PaastaColors.red(
+                f"Automated redeploys not enabled for {service}, returning no image_version"
+            ),
+            file=sys.stderr,
+        )
+        return 0
+
+    current_timestamp = datetime.datetime.now()
+
+    # TODO: Handle additional identifiers, flavor, etc
+    new_image_version = format_timestamp(current_timestamp)
+
+    if args.force:
+        # Force a new version immediately
+        print(new_image_version)
+        return 0
+
+    # get latest image_version of any deploy group from deployments.json
+    deployments = load_v2_deployments_json(service, soa_dir)
+    latest_image_version = get_latest_image_version(
+        deployments.config_dict, commit=args.commit
+    )
+
+    if should_generate_new_image_version(
+        old=latest_image_version, new=new_image_version, max_age=args.max_age
+    ):
+        print(new_image_version)
+    else:
+        print(latest_image_version)
+
+    return 0

paasta_tools/cli/cmds/get_latest_deployment.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import sys
+
+from paasta_tools.cli.utils import lazy_choices_completer
+from paasta_tools.cli.utils import list_deploy_groups
+from paasta_tools.cli.utils import PaastaColors
+from paasta_tools.cli.utils import validate_service_name
+from paasta_tools.deployment_utils import get_currently_deployed_version
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import list_services
+
+
+def add_subparser(subparsers):
+    list_parser = subparsers.add_parser(
+        "get-latest-deployment",
+        help="Gets the Git SHA for the latest deployment of a service",
+    )
+    list_parser.add_argument(
+        "-s",
+        "--service",
+        help="Name of the service which you want to get the latest deployment for.",
+        required=True,
+    ).completer = lazy_choices_completer(list_services)
+    list_parser.add_argument(
+        "-i",
+        "-l",
+        "--deploy-group",
+        help="Name of the deploy group which you want to get the latest deployment for.",
+        required=True,
+    ).completer = lazy_choices_completer(list_deploy_groups)
+    list_parser.add_argument(
+        "-d",
+        "--soa-dir",
+        help="A directory from which soa-configs should be read from",
+        default=DEFAULT_SOA_DIR,
+    )
+    format_group = list_parser.add_mutually_exclusive_group()
+    format_group.add_argument(
+        "--sha-only",
+        help="Return only the latest sha for this deploy group, not the full deployed version",
+        action="store_true",
+        default=False,
+    )
+    format_group.add_argument(
+        "-j",
+        "--json",
+        help="Return result in json format instead of raw string",
+        action="store_true",
+        default=False,
+    )
+
+    list_parser.set_defaults(command=paasta_get_latest_deployment)
+
+
+def paasta_get_latest_deployment(args):
+    service = args.service
+    deploy_group = args.deploy_group
+    soa_dir = args.soa_dir
+    validate_service_name(service, soa_dir)
+
+    version = get_currently_deployed_version(
+        service=service, deploy_group=deploy_group, soa_dir=soa_dir
+    )
+    if not version:
+        print(
+            PaastaColors.red(
+                f"A deployment could not be found for {deploy_group} in {service}"
+            ),
+            file=sys.stderr,
+        )
+        return 1
+    else:
+        if args.sha_only:
+            print(version.sha)
+        else:
+            if args.json:
+                print(version.json())
+            else:
+                print(version)
+        return 0

paasta_tools/cli/cmds/info.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Collection
+from typing import List
+
+from service_configuration_lib import read_service_configuration
+
+from paasta_tools.cli.cmds.status import get_actual_deployments
+from paasta_tools.cli.utils import figure_out_service_name
+from paasta_tools.cli.utils import lazy_choices_completer
+from paasta_tools.long_running_service_tools import get_all_namespaces_for_service
+from paasta_tools.long_running_service_tools import load_service_namespace_config
+from paasta_tools.monitoring_tools import get_runbook
+from paasta_tools.monitoring_tools import get_team
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import get_git_url
+from paasta_tools.utils import list_services
+from paasta_tools.utils import NoDeploymentsAvailable
+from paasta_tools.utils import PaastaColors
+
+NO_DESCRIPTION_MESSAGE = "No 'description' entry in service.yaml. Please add a one line sentence that describes this service"
+NO_EXTERNAL_LINK_MESSAGE = (
+    "No 'external_link' entry in service.yaml. "
+    "Please add one that points to a reference doc for your service"
+)
+
+# modes that depend on smartstack port cannot be tested via paasta proxies, so we exclude those
+TESTABLE_SERVICE_MODES = {"http", "https"}
+
+
+def add_subparser(subparsers):
+    list_parser = subparsers.add_parser(
+        "info",
+        help="Prints the general information about a service.",
+        description=(
+            "'paasta info' gathers information about a service from soa-configs "
+            "and prints it in a human-friendly way. It does no API calls, it "
+            "just analyzes the config files."
+        ),
+    )
+    list_parser.add_argument(
+        "-s", "--service", help="The name of the service you wish to inspect"
+    ).completer = lazy_choices_completer(list_services)
+    list_parser.add_argument(
+        "-d",
+        "--soa-dir",
+        dest="soa_dir",
+        metavar="SOA_DIR",
+        default=DEFAULT_SOA_DIR,
+        help="define a different soa config directory",
+    )
+    list_parser.set_defaults(command=paasta_info)
+
+
+def deployments_to_clusters(deployments: Collection[str]) -> Collection[str]:
+    clusters = []
+    for deployment in deployments:
+        cluster = deployment.split(".")[0]
+        clusters.append(cluster)
+    return set(clusters)
+
+
+def get_smartstack_endpoints(service, soa_dir):
+    endpoints = []
+    for name, config in get_all_namespaces_for_service(
+        service, full_name=False, soa_dir=soa_dir
+    ):
+        mode = config.get("mode", "http")
+        port = config.get("proxy_port")
+        endpoints.append(f"{mode}://169.254.255.254:{port} ({name})")
+    return endpoints
+
+
+def get_deployments_strings(service: str, soa_dir: str) -> List[str]:
+    output = []
+    try:
+        deployments = get_actual_deployments(service, soa_dir)
+    except NoDeploymentsAvailable:
+        deployments = {}
+    if deployments == {}:
+        output.append(" - N/A: Not deployed to any PaaSTA Clusters")
+    else:
+        service_config = load_service_namespace_config(
+            service=service, namespace="main", soa_dir=soa_dir
+        )
+        service_mode = service_config.get_mode()
+        for cluster in deployments_to_clusters(deployments):
+            if service_mode in TESTABLE_SERVICE_MODES:
+                link = PaastaColors.cyan(
+                    f"{service_mode}://{service}.proxy.{cluster}.paasta/"
+                )
+            else:
+                link = "N/A"
+            output.append(f" - {cluster} ({link})")
+    return output
+
+
+def get_dashboard_urls(service):
+    output = [
+        " - %s (service load y/sl2)" % (PaastaColors.cyan(f"http://y/{service}_load"))
+    ]
+    return output
+
+
+def get_service_info(service, soa_dir):
+    service_configuration = read_service_configuration(service, soa_dir)
+    description = service_configuration.get("description", NO_DESCRIPTION_MESSAGE)
+    external_link = service_configuration.get("external_link", NO_EXTERNAL_LINK_MESSAGE)
+    smartstack_endpoints = get_smartstack_endpoints(service, soa_dir)
+    git_url = get_git_url(service, soa_dir)
+
+    output = []
+    output.append("Service Name: %s" % service)
+    output.append("Description: %s" % description)
+    output.append("External Link: %s" % PaastaColors.cyan(external_link))
+    output.append(
+        "Monitored By: team %s"
+        % get_team(service=service, overrides={}, soa_dir=soa_dir)
+    )
+    output.append(
+        "Runbook: %s"
+        % PaastaColors.cyan(get_runbook(service=service, overrides={}, soa_dir=soa_dir))
+    )
+    output.append("Git Repo: %s" % git_url)
+    output.append(
+        "Deployed to the following clusters (with test URLs, where available):"
+    )
+    output.extend(get_deployments_strings(service, soa_dir))
+    if smartstack_endpoints:
+        output.append("Smartstack endpoint(s):")
+        for endpoint in smartstack_endpoints:
+            output.append(" - %s" % endpoint)
+    output.append("Dashboard(s):")
+    output.extend(get_dashboard_urls(service))
+
+    return "\n".join(output)
+
+
+def paasta_info(args):
+    """Prints general information about a service"""
+    soa_dir = args.soa_dir
+    service = figure_out_service_name(args, soa_dir=soa_dir)
+    print(get_service_info(service, soa_dir))

paasta_tools/cli/cmds/itest.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+from paasta_tools.cli.utils import get_jenkins_build_output_url
+from paasta_tools.cli.utils import lazy_choices_completer
+from paasta_tools.cli.utils import validate_service_name
+from paasta_tools.utils import _log
+from paasta_tools.utils import _run
+from paasta_tools.utils import build_docker_tag
+from paasta_tools.utils import check_docker_image
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import list_services
+
+
+def add_subparser(subparsers):
+    list_parser = subparsers.add_parser(
+        "itest",
+        help="Runs 'make itest' as part of the PaaSTA contract.",
+        description=(
+            "'paasta itest' runs 'make itest' in the root of a service directory. "
+            "It is designed to be used in conjunction with the 'Jenkins' workflow: "
+            "http://paasta.readthedocs.io/en/latest/about/contract.html#jenkins-pipeline-recommended"
+        ),
+    )
+    list_parser.add_argument(
+        "-s",
+        "--service",
+        help="Test and build docker image for this service. Leading "
+        '"services-", as included in a Jenkins job name, '
+        "will be stripped.",
+        required=True,
+    )
+    list_parser.add_argument(
+        "-c",
+        "--commit",
+        help="Git sha used to construct tag for built image",
+        required=True,
+    )
+    list_parser.add_argument(
+        "--image-version",
+        type=str,
+        required=False,
+        default=None,
+        help="Extra version metadata used to construct tag for built image",
+    )
+    list_parser.add_argument(
+        "-d",
+        "--soa-dir",
+        dest="soa_dir",
+        help="A directory from which soa-configs should be read from",
+        default=DEFAULT_SOA_DIR,
+    ).completer = lazy_choices_completer(list_services)
+    list_parser.add_argument(
+        "--timeout",
+        dest="timeout",
+        help="How many seconds before this command times out",
+        default=3600,
+        type=float,
+    )
+    list_parser.set_defaults(command=paasta_itest)
+
+
+def paasta_itest(args):
+    """Build and test a docker image"""
+    service = args.service
+    soa_dir = args.soa_dir
+    if service and service.startswith("services-"):
+        service = service.split("services-", 1)[1]
+    validate_service_name(service, soa_dir=soa_dir)
+
+    tag = build_docker_tag(service, args.commit, args.image_version)
+    run_env = os.environ.copy()
+    run_env["DOCKER_TAG"] = tag
+    cmd = "make itest"
+    loglines = []
+
+    _log(
+        service=service,
+        line="starting itest for %s." % args.commit,
+        component="build",
+        level="event",
+    )
+    returncode, output = _run(
+        cmd,
+        env=run_env,
+        timeout=args.timeout,
+        log=True,
+        component="build",
+        service=service,
+        loglevel="debug",
+    )
+    if returncode != 0:
+        loglines.append("ERROR: itest failed for %s." % args.commit)
+        output = get_jenkins_build_output_url()
+        if output:
+            loglines.append("See output: %s" % output)
+    else:
+        loglines.append("itest passed for %s." % args.commit)
+        if not check_docker_image(service, args.commit, args.image_version):
+            loglines.append("ERROR: itest has not created %s" % tag)
+            returncode = 1
+    for logline in loglines:
+        _log(service=service, line=logline, component="build", level="event")
+    return returncode

paasta_tools/cli/cmds/list.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from service_configuration_lib import DEFAULT_SOA_DIR
+
+from paasta_tools.cli.utils import list_paasta_services
+from paasta_tools.cli.utils import list_service_instances
+from paasta_tools.utils import list_services
+from paasta_tools.utils import SPACER
+
+
+def add_subparser(subparsers):
+    list_parser = subparsers.add_parser(
+        "list",
+        help="Display a list of PaaSTA services",
+        description=(
+            "'paasta list' inspects the soa-configs directory and lists all of the "
+            "PaaSTA services that are declared."
+        ),
+    )
+    list_parser.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        help="Display all services, even if not on PaaSTA.",
+    )
+    list_parser.add_argument(
+        "-i",
+        "--print-instances",
+        action="store_true",
+        help="Display all service%sinstance values, which only PaaSTA services have."
+        % SPACER,
+    )
+    list_parser.add_argument(
+        "-y",
+        "--yelpsoa-config-root",
+        dest="soa_dir",
+        help="A directory from which yelpsoa-configs should be read from",
+        default=DEFAULT_SOA_DIR,
+    )
+    list_parser.set_defaults(command=paasta_list)
+
+
+def paasta_list(args):
+    """Print a list of Yelp services currently running
+    :param args: argparse.Namespace obj created from sys.args by cli"""
+    if args.print_instances:
+        services = list_service_instances(args.soa_dir)
+    elif args.all:
+        services = list_services(args.soa_dir)
+    else:
+        services = list_paasta_services(args.soa_dir)
+
+    for service in services:
+        print(service)

paasta_tools/cli/cmds/list_clusters.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import list_clusters
+
+
+def add_subparser(subparsers):
+    list_parser = subparsers.add_parser(
+        "list-clusters",
+        help="Display a list of all PaaSTA clusters",
+        description=(
+            "'paasta list' inspects all of the PaaSTA services declared in the soa-configs "
+            "directory, and prints the set of unique clusters that are used.\n\n"
+            "The command can only report those clusters that are actually used by some services."
+        ),
+    )
+    list_parser.add_argument(
+        "-d",
+        "--soa-dir",
+        dest="soa_dir",
+        metavar="SOA_DIR",
+        default=DEFAULT_SOA_DIR,
+        help="define a different soa config directory",
+    )
+    list_parser.set_defaults(command=paasta_list_clusters)
+
+
+def paasta_list_clusters(args, **kwargs):
+    for cluster in list_clusters(soa_dir=args.soa_dir):
+        print(cluster)