paasta-tools 1.21.3__py3-none-any.whl

paasta_tools/contrib/service_shard_update.py

@@ -0,0 +1,373 @@
+import argparse
+import logging
+import sys
+
+from paasta_tools.cli.utils import trigger_deploys
+from paasta_tools.config_utils import AutoConfigUpdater
+from paasta_tools.utils import DEFAULT_SOA_CONFIGS_GIT_URL
+from paasta_tools.utils import format_git_url
+from paasta_tools.utils import load_system_paasta_config
+
+log = logging.getLogger(__name__)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="")
+    parser.add_argument(
+        "--git-remote",
+        help="Master git repo for soaconfigs",
+        default=None,
+        dest="git_remote",
+    )
+    parser.add_argument(
+        "--branch",
+        help="Branch name to push to",
+        required=True,
+        dest="branch",
+    )
+    parser.add_argument(
+        "--local-dir",
+        help="Act on configs in the local directory rather than cloning the git_remote",
+        required=False,
+        default=None,
+        dest="local_dir",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        help="Logging verbosity",
+        action="store_true",
+        dest="verbose",
+    )
+    parser.add_argument(
+        "-d",
+        "--dry-run",
+        help="Do not commit changes to git",
+        action="store_true",
+        dest="dry_run",
+    )
+    parser.add_argument(
+        "--source-id",
+        help="String to attribute the changes in the commit message.",
+        required=False,
+        default=None,
+        dest="source_id",
+    )
+    parser.add_argument(
+        "--service",
+        help="Service to modify",
+        required=True,
+        dest="service",
+    )
+    parser.add_argument(
+        "--min-instance-count",
+        help="If a deploy group is added, the min_instance count to create it with",
+        required=False,
+        default=1,
+        dest="min_instance_count",
+    )
+    parser.add_argument(
+        "--prod-max-instance-count",
+        help="If a deploy group is added, the prod max_instance count to create it with",
+        required=False,
+        default=100,
+        type=int,
+        dest="prod_max_instance_count",
+    )
+    parser.add_argument(
+        "--non-prod-max-instance-count",
+        help="If a deploy group is added, the non-prod max_instance count to create it with",
+        required=False,
+        default=5,
+        type=int,
+        dest="non_prod_max_instance_count",
+    )
+    parser.add_argument(
+        "--cpus",
+        help="If a deploy group is added, the cpu value to create it with",
+        required=False,
+        type=float,
+        dest="cpus",
+    )
+    parser.add_argument(
+        "--mem",
+        help="If a deploy group is added, the mem value to create it with",
+        required=False,
+        type=int,
+        dest="mem",
+    )
+    parser.add_argument(
+        "--setpoint",
+        help="If a deploy group is added, the autoscaling.setpoint value to create it with",
+        required=False,
+        type=float,
+        dest="setpoint",
+    )
+    parser.add_argument(
+        "--shard-name",
+        help="Shard name to add if it does not exist",
+        required=True,
+        dest="shard_name",
+    )
+    parser.add_argument(
+        "--metrics-provider",
+        help="Autoscaling metrics provider",
+        required=False,
+        dest="metrics_provider",
+    )
+    parser.add_argument(
+        "--timeout-server-ms",
+        help="smartstack server timeout",
+        required=False,
+        type=int,
+        dest="timeout_server_ms",
+    )
+    parser.add_argument(
+        "--autotune-min-cpus",
+        help="Minimum number of CPUs Autotune should give the shard",
+        required=False,
+        type=float,
+        dest="autotune_min_cpus",
+    )
+    parser.add_argument(
+        "--autotune-max-cpus",
+        help="Maximum number of CPUs Autotune should give the shard",
+        required=False,
+        type=float,
+        dest="autotune_max_cpus",
+    )
+    parser.add_argument(
+        "--autotune-min-mem",
+        help="Minimum amount of memory Autotune should give the shard",
+        required=False,
+        type=int,
+        dest="autotune_min_mem",
+    )
+    parser.add_argument(
+        "--autotune-max-mem",
+        help="Maximum amount of memory Autotune should give the shard",
+        required=False,
+        type=int,
+        dest="autotune_max_mem",
+    )
+    parser.add_argument(
+        "--autotune-min-disk",
+        help="Minimum amount of disk Autotune should give the shard",
+        required=False,
+        type=int,
+        dest="autotune_min_disk",
+    )
+    parser.add_argument(
+        "--autotune-max-disk",
+        help="Maximum amount of disk Autotune should give the shard",
+        required=False,
+        type=int,
+        dest="autotune_max_disk",
+    )
+    parser.add_argument(
+        "--iam-role",
+        help="IAM role to use for the shard",
+        required=False,
+        type=str,
+        dest="iam_role",
+    )
+    parser.add_argument(
+        "--environment",
+        help="Environment to deploy in, defaults to all environments if not specified",
+        required=False,
+        choices=DEPLOY_MAPPINGS.keys(),
+        dest="environment",
+    )
+    return parser.parse_args()
+
+
+def get_default_git_remote():
+    system_paasta_config = load_system_paasta_config()
+    repo_config = system_paasta_config.get_git_repo_config("yelpsoa-configs")
+    default_git_remote = format_git_url(
+        system_paasta_config.get_git_config()["git_user"],
+        repo_config.get("git_server", DEFAULT_SOA_CONFIGS_GIT_URL),
+        repo_config["repo_name"],
+    )
+    return default_git_remote
+
+
+DEPLOY_MAPPINGS = {
+    "dev": ["norcal-devc"],
+    "stage": ["norcal-stagef", "norcal-stageg"],
+    "prod": ["nova-prod", "pnw-prod"],
+}
+
+
+def main(args):
+    changes_made = False
+    updater = AutoConfigUpdater(
+        config_source=args.source_id,
+        git_remote=args.git_remote or get_default_git_remote(),
+        branch=args.branch,
+        working_dir=args.local_dir or "/nail/tmp",
+        do_clone=args.local_dir is None,
+    )
+    deploy_environments = (
+        {args.environment: DEPLOY_MAPPINGS[args.environment]}
+        if args.environment
+        else DEPLOY_MAPPINGS
+    )
+
+    with updater:
+        deploy_file = updater.get_existing_configs(args.service, "deploy")
+        smartstack_file = updater.get_existing_configs(args.service, "smartstack")
+        shard_deploy_groups = {
+            f"{prefix}.{args.shard_name}" for prefix in deploy_environments.keys()
+        }
+        pipeline_steps = {step["step"] for step in deploy_file["pipeline"]}
+
+        if not shard_deploy_groups.issubset(pipeline_steps):
+            changes_made = True
+            steps_to_add = shard_deploy_groups - pipeline_steps
+
+            # If the pipeline does not contain deploy groups for the service shard
+            # Add the missing steps and write to deploy config
+            for step in steps_to_add:
+                deploy_file["pipeline"].append(
+                    {
+                        "step": step,
+                        "wait_for_deployment": True,
+                        "disabled": True,
+                    }
+                )
+                log.info(f"{step} added to deploy config")
+            updater.write_configs(args.service, "deploy", deploy_file)
+
+            for deploy_prefix, config_paths in deploy_environments.items():
+                for config_path in config_paths:
+                    # Determine configuration suffix (PAASTA-18216)
+                    eks_config = updater.get_existing_configs(
+                        args.service, f"eks-{config_path}"
+                    )
+                    kube_config = updater.get_existing_configs(
+                        args.service, f"kubernetes-{config_path}"
+                    )
+
+                    if eks_config:
+                        config_file = eks_config
+                        config_prefix = "eks-"
+                    elif kube_config:
+                        config_file = kube_config
+                        config_prefix = "kubernetes-"
+                    else:
+                        log.error(
+                            f"No EKS or Kubernetes config found for {args.service}"
+                        )
+                        continue
+
+                    instance_config = {
+                        "deploy_group": f"{deploy_prefix}.{args.shard_name}",
+                        "min_instances": args.min_instance_count,
+                        "max_instances": (
+                            args.prod_max_instance_count
+                            if deploy_prefix == "prod"
+                            else args.non_prod_max_instance_count
+                        ),
+                        "env": {
+                            "PAASTA_SECRET_BUGSNAG_API_KEY": "SECRET(bugsnag_api_key)",
+                        },
+                    }
+                    if args.metrics_provider is not None or args.setpoint is not None:
+                        instance_config["autoscaling"] = {"metrics_providers": []}
+                        metrics_provider_config = {}
+                        if args.metrics_provider is not None:
+                            metrics_provider_config["type"] = args.metrics_provider
+                        if args.setpoint is not None:
+                            metrics_provider_config["setpoint"] = args.setpoint
+                        instance_config["autoscaling"]["metrics_providers"].append(
+                            metrics_provider_config
+                        )
+
+                    if args.iam_role is not None:
+                        instance_config["iam_role"] = args.iam_role
+                    if args.cpus is not None:
+                        instance_config["cpus"] = args.cpus
+                    if args.mem is not None:
+                        instance_config["mem"] = args.mem
+                    if any(
+                        (
+                            args.autotune_min_cpus,
+                            args.autotune_max_cpus,
+                            args.autotune_min_mem,
+                            args.autotune_max_mem,
+                            args.autotune_min_disk,
+                            args.autotune_max_disk,
+                        )
+                    ):
+                        limit_config = {}
+                        limit_config["cpus"] = {
+                            "min": args.autotune_min_cpus,
+                            "max": args.autotune_max_cpus,
+                        }
+                        limit_config["mem"] = {
+                            "min": args.autotune_min_mem,
+                            "max": args.autotune_max_mem,
+                        }
+                        limit_config["disk"] = {
+                            "min": args.autotune_min_disk,
+                            "max": args.autotune_max_disk,
+                        }
+
+                        # remove any None values to keep the config clean
+                        for resource in list(limit_config):
+                            for key in list(limit_config[resource]):
+                                if limit_config[resource][key] is None:
+                                    del limit_config[resource][key]
+                            if len(limit_config[resource]) == 0:
+                                del limit_config[resource]
+
+                        if len(limit_config) > 0:
+                            instance_config["autotune_limits"] = limit_config
+                    # If the service config does not contain definitions for the shard in each ecosystem
+                    # Add the missing definition and write to the corresponding config
+                    if args.shard_name not in config_file.keys():
+                        config_file[args.shard_name] = instance_config
+
+                        updater.write_configs(
+                            args.service, f"{config_prefix}{config_path}", config_file
+                        )
+                        log.info(
+                            f"{deploy_prefix}.{args.shard_name} added to {config_prefix}{config_path}"
+                        )
+        else:
+            log.info(f"{args.shard_name} is in deploy config already.")
+
+        # If the service shard is not defined in smartstack
+        # Add the definition with a suggested proxy port
+        if args.shard_name not in smartstack_file.keys():
+            changes_made = True
+            smartstack_file[args.shard_name] = {
+                "proxy_port": None,
+                "extra_advertise": {"ecosystem:devc": ["ecosystem:devc"]},
+            }
+            if args.timeout_server_ms:
+                smartstack_file[args.shard_name][
+                    "timeout_server_ms"
+                ] = args.timeout_server_ms
+            updater.write_configs(args.service, "smartstack", smartstack_file)
+        else:
+            log.info(f"{args.shard_name} is in smartstack config already, skipping.")
+
+        # Only commit to remote if changes were made
+        if changes_made and not args.dry_run:
+            updater.commit_to_remote()
+            trigger_deploys(args.service)
+        else:
+            # exit with code to indicate nothing was changed
+            sys.exit(129)
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.WARNING)
+
+    main(args)

paasta_tools/contrib/shared_ip_check.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3.8
+import sys
+from collections import defaultdict
+
+import iptc
+
+from paasta_tools import iptables
+from paasta_tools.utils import get_docker_client
+
+
+def list_docker_nat_rules():
+    chain_name = "DOCKER"
+    table = iptc.Table(iptc.Table.NAT)
+    chain = iptc.Chain(table, chain_name)
+    for rule in chain.rules:
+        yield iptables.Rule.from_iptc(rule)
+
+
+def main():
+    docker_client = get_docker_client()
+    ip_to_containers = defaultdict(list)
+
+    for container in docker_client.containers():
+        networks = container["NetworkSettings"]["Networks"]
+        if "bridge" in networks:
+            ip = networks["bridge"]["IPAddress"]
+            if ip:
+                ip_to_containers[ip].append(container)
+
+    output = []
+    for ip, containers in ip_to_containers.items():
+        if len(containers) > 1:
+            output.append(f"{ip} shared by the following containers:")
+            for container in containers:
+                output.append("    Image: {}".format(container["Image"]))
+                output.append("        ID: {}".format(container["Id"]))
+                output.append("        State: {}".format(container["State"]))
+                output.append("        Status: {}".format(container["Status"]))
+            output.append("")
+
+    if output:
+        print(
+            "CRITICAL - There are multiple Docker containers assigned to the same IP."
+        )
+        print(
+            "There should only be one per IP. Choose one to keep and try stopping the others."
+        )
+        print("\n".join(output))
+        return 2
+    else:
+        print("OK - No Docker containers sharing an IP on this host.")
+
+    targets_seen = {}
+    duplicates_found = False
+    for rule in list_docker_nat_rules():
+        target = rule.target_parameters
+        if target not in targets_seen:
+            targets_seen[target] = rule
+        else:
+            print(
+                "This is the second time we've seen a rule with the same target_parameters!"
+            )
+            print(rule)
+            print("The other rule with that target is:")
+            print(targets_seen[target])
+            duplicates_found = True
+    if duplicates_found is True:
+        print(
+            "CRITICAL - Duplicate iptables rules found! This will route traffic to the wrong service!"
+        )
+        return 2
+    else:
+        print("OK - No duplicate Docker iptables rules detected")
+
+
+if __name__ == "__main__":
+    sys.exit(main())

paasta_tools/contrib/timeouts_metrics_prom.py

@@ -0,0 +1,64 @@
+#!/opt/venvs/paasta-tools/bin/python
+"""
+The goal is to walk through all smartstack.yaml in yelpsoa directories
+and convert timeout values to the prometheus format.
+
+For every upstream and path, if endpoint timeout is specified, we record
+per endpoint timeout value. If timeout_server_ms is specified,
+we record timeout_server_ms value in path `/`. Otherwise, we record the default
+timeout_server_ms 1000.
+
+Example output:
+# HELP endpoint timeout value defined in yelpsoa-config.
+# TYPE TYPE yelpsoaconfig_endpoint_timeouts_ms gauge
+yelpsoaconfig_endpoint_timeouts_ms{path="/consumer_app/devices/braze/info",upstream="push_notifications.main.egress_cluster"} 10000.0
+yelpsoaconfig_endpoint_timeouts_ms{path="/",upstream="mysql_read_security.main.egress_cluster"} 100.0
+"""
+import os
+
+from prometheus_client import CollectorRegistry
+from prometheus_client import Gauge
+from prometheus_client import write_to_textfile
+from prometheus_client.metrics import MetricWrapperBase
+
+from paasta_tools import yaml_tools as yaml
+from paasta_tools.utils import DEFAULT_SOA_DIR
+
+PROM_OUTPUT_FILE = f"{DEFAULT_SOA_DIR}/.autotune_timeouts.prom"
+DEFAULT_TIMEOUT = 1000
+
+
+def read_and_write_timeouts_metrics(
+    root: str, service: str, prom_metric: MetricWrapperBase
+) -> None:
+    with open(os.path.join(root, "smartstack.yaml")) as smartstack_file:
+        smartstack_yaml = yaml.safe_load(smartstack_file)
+    for instance_name, info in smartstack_yaml.items():
+        upstream = f"{service}.{instance_name}.egress_cluster"
+        if "endpoint_timeouts" in info:
+            for path, endpoint_timeout in info["endpoint_timeouts"].items():
+                prom_metric.labels(path, upstream).set(endpoint_timeout)
+        # always record default timeout
+        default_timeout = info.get("timeout_server_ms", DEFAULT_TIMEOUT)
+        prom_metric.labels("/", upstream).set(default_timeout)
+
+
+if __name__ == "__main__":
+    registry = CollectorRegistry()
+    prom_metric = Gauge(
+        "yelpsoaconfig_endpoint_timeouts_ms",
+        "endpoint timeout value defined in yelpsoa-config",
+        ["path", "upstream"],
+        registry=registry,
+    )
+    # Walk through soa config dir and filter smartstack yaml
+    for root, dirs, files in os.walk(DEFAULT_SOA_DIR):
+        service = root.split("/")[-1]
+        # Avoid confusion of the smartstacks.yaml under autotuned_defaults/ in the future
+        if "autotuned_defaults" == service:
+            continue
+        for f in files:
+            if f == "smartstack.yaml":
+                read_and_write_timeouts_metrics(root, service, prom_metric)
+
+    write_to_textfile(PROM_OUTPUT_FILE, registry)

paasta_tools/delete_kubernetes_deployments.py

@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage: ./delete_paasta_contract_monitor.py <service.instance> [options]
+
+The following script is a setup on a cron job in k8s masters. This is responsible for deleting
+paasta-contract-monitor deployments and its services. By deleting the deployment itself,
+setup_kubernetes_job.py will be able to reschedule the deployment and its pods on different nodes.
+"""
+import argparse
+import logging
+import sys
+
+from paasta_tools.kubernetes_tools import delete_deployment
+from paasta_tools.kubernetes_tools import ensure_namespace
+from paasta_tools.kubernetes_tools import get_kubernetes_app_name
+from paasta_tools.kubernetes_tools import KubeClient
+from paasta_tools.utils import decompose_job_id
+from paasta_tools.utils import InvalidJobNameError
+from paasta_tools.utils import SPACER
+
+log = logging.getLogger(__name__)
+
+
+def parse_args(args=None) -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Deletes list of deployments.")
+    parser.add_argument(
+        "service_instance_list",
+        nargs="+",
+        help="The list of service instances to delete",
+        metavar=f"SERVICE{SPACER}INSTANCE",
+    )
+    args = parser.parse_args(args=args)
+    return args
+
+
+def get_deployment_names_from_list(service_instance_list):
+    app_names = []
+    for service_instance in service_instance_list:
+        try:
+            service, instance, _, __ = decompose_job_id(service_instance)
+            app_name = get_kubernetes_app_name(service, instance)
+            app_names.append(app_name)
+        except InvalidJobNameError:
+            log.error(
+                f"Invalid service instance specified. Format is service{SPACER}instance."
+            )
+            sys.exit(1)
+    return app_names
+
+
+def main(args=None) -> None:
+    args = parse_args(args=args)
+    service_instance_list = args.service_instance_list
+    deployment_names = get_deployment_names_from_list(service_instance_list)
+
+    log.debug(f"Deleting deployments: {deployment_names}")
+    kube_client = KubeClient()
+    ensure_namespace(kube_client=kube_client, namespace="paasta")
+
+    for deployment_name in deployment_names:
+        try:
+            log.debug(f"Deleting {deployment_name}")
+            delete_deployment(
+                kube_client=kube_client,
+                deployment_name=deployment_name,
+                namespace="paasta",
+            )
+        except Exception as err:
+            log.error(f"Unable to delete {deployment_name}: {err}")
+            sys.exit(1)
+
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

paasta_tools/deployment_utils.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Optional
+
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import DeploymentVersion
+from paasta_tools.utils import load_v2_deployments_json
+from paasta_tools.utils import NoDeploymentsAvailable
+
+
+def get_currently_deployed_sha(service, deploy_group, soa_dir=DEFAULT_SOA_DIR):
+    """Tries to determine the currently deployed sha for a service and deploy_group,
+    returns None if there isn't one ready yet"""
+    try:
+        deployments = load_v2_deployments_json(service=service, soa_dir=soa_dir)
+        return deployments.get_git_sha_for_deploy_group(deploy_group=deploy_group)
+    except NoDeploymentsAvailable:
+        return None
+
+
+def get_currently_deployed_version(
+    service, deploy_group, soa_dir=DEFAULT_SOA_DIR
+) -> Optional[DeploymentVersion]:
+    """Tries to determine the currently deployed version for a service and deploy_group,
+    returns None if there isn't one ready yet"""
+    try:
+        deployments = load_v2_deployments_json(service=service, soa_dir=soa_dir)
+        return deployments.get_deployment_version_for_deploy_group(
+            deploy_group=deploy_group
+        )
+    except NoDeploymentsAvailable:
+        return None