paasta-tools 1.21.3__py3-none-any.whl

paasta_tools/contrib/render_template.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3.8
+import argparse
+import os
+import re
+
+from paasta_tools import yaml_tools as yaml
+
+
+def replace(s, values):
+    s = re.sub(
+        r"<%(.*?)%>",
+        lambda x: values.get(
+            x.group(0).replace("<%", "").replace("%>", ""), x.group(0)
+        ),
+        s,
+    )
+    return re.sub(
+        r"\$\((.*?)\)",
+        lambda x: os.environ.get(
+            x.group(0).replace("$(", "").replace(")", ""), x.group(0)
+        ),
+        s,
+    )
+
+
+def render_file(src, dst, values, overwrite=True):
+    basename = os.path.basename(src)
+    new_name = replace(basename, values)
+    if not os.path.exists(new_name) or overwrite:
+        write_file(dst, new_name, src, values)
+
+
+def write_file(dst, name, src, values):
+    with open(f"{dst}/{name}", "w") as new:
+        with open(f"{src}", "r") as old:
+            new.write(replace(old.read(), values))
+
+
+def render(src, dst, values={}, exclude={}, overwrite=True):
+    if os.path.isfile(src):
+        render_file(src, dst, values, overwrite)
+        return
+    for f in os.scandir(src):
+        if f.name.startswith(".") or f.path in exclude:
+            continue
+        if os.path.isfile(f.path):
+            render_file(f.path, dst, values, overwrite)
+        else:
+            new_dst = replace(f"{dst}/{f.name}", values)
+            try:
+                os.makedirs(new_dst, exist_ok=True)
+            except OSError as e:
+                if e.errno != os.errno.EEXIST:
+                    raise
+            render(f.path, new_dst, values, exclude, overwrite)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Replaces all <%%> in all files in src with values provided, and writes the results to dst folder. $() is reserved for environment variables. File/dir that starts with . are ignored"
+    )
+    parser.add_argument(
+        "-s",
+        "--src",
+        type=str,
+        dest="src",
+        required=True,
+        help="src can be either a valid folder of directory. Note that src directory itself is not rendered. .* files/dirs are ignored.",
+    )
+    parser.add_argument(
+        "-d",
+        "--dst",
+        type=str,
+        dest="dst",
+        required=True,
+        help="Dst needs to be a directory.",
+    )
+    parser.add_argument(
+        "-v",
+        "--values",
+        type=str,
+        dest="values",
+        default=None,
+        help="values need to be valid file if provided",
+    )
+    args = parser.parse_args()
+    return args
+
+
+def render_values(src: str, dst: str, values: str, overwrite=True) -> None:
+    if values is not None:
+        values = os.path.abspath(values)
+    # Validate src and values. Dst needs to be a directory. src can be either a valid folder of directory. values need to be valid file if provided.
+    if not os.path.exists(src):
+        raise Exception("src path is invalid")
+    if not os.path.exists(dst) or not os.path.isdir(dst):
+        raise Exception("dst path is invalid")
+    if values and (not os.path.exists(values) or not os.path.isfile(values)):
+        raise Exception("values path is invalid")
+    # Lookup for values.yaml in src folder if values is not provided
+    if os.path.isdir(src) and values is None and os.path.exists(f"{src}/values.yaml"):
+        values = f"{src}/values.yaml"
+    config_dict = {}
+    if values is not None:
+        with open(values) as f:
+            config_dict = yaml.safe_load(f)
+    # Replace environment variables in values.yaml with environment variables
+    for k, v in config_dict.items():
+        config_dict[k] = re.sub(
+            r"\$\((.*?)\)",
+            lambda x: os.environ.get(
+                x.group(0).replace("$(", "").replace(")", ""), x.group(0)
+            ),
+            v,
+        )
+    render(src, dst, config_dict, {values}, overwrite)
+
+
+def main():
+    args = parse_args()
+    src = os.path.abspath(args.src)
+    dst = os.path.abspath(args.dst)
+    values = args.values
+
+    render_values(src, dst, values)
+
+
+if __name__ == "__main__":
+    main()

paasta_tools/contrib/rightsizer_soaconfigs_update.py

@@ -0,0 +1,348 @@
+import argparse
+import logging
+from collections import defaultdict
+from typing import Any
+from typing import cast
+from typing import Dict
+from typing import List
+from typing import Literal
+from typing import Optional
+from typing import Set
+from typing import TypedDict
+from typing import Union
+
+from paasta_tools.config_utils import AutoConfigUpdater
+from paasta_tools.contrib.paasta_update_soa_memcpu import get_report_from_splunk
+from paasta_tools.kubernetes_tools import SidecarResourceRequirements
+from paasta_tools.utils import AUTO_SOACONFIG_SUBDIR
+from paasta_tools.utils import DEFAULT_SOA_CONFIGS_GIT_URL
+from paasta_tools.utils import format_git_url
+from paasta_tools.utils import load_system_paasta_config
+
+
+log = logging.getLogger(__name__)
+
+NULL = "null"
+SUPPORTED_CSV_KEYS = (
+    "cpus",
+    "mem",
+    "disk",
+    "hacheck_cpus",
+    "cpu_burst_add",
+    "min_instances",
+    "max_instances",
+)
+HEADER_COMMENT = """
+# This file contains recommended config values for your service generated by
+# automated processes.
+#
+# Your service will use these values if they are not defined in
+# {regular_filename}.
+#
+# If you would like to override a config value defined here, add the config
+# value to {regular_filename} instead of editing this file.
+# ==============================================================================
+{{}}
+"""
+# ^ Needs an empty dict at the end for ruamel to return a non-None value when loading
+# ^ Braces are doubled for escaping in call to .format
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="")
+    parser.add_argument(
+        "-s",
+        "--splunk-creds",
+        help="Service credentials for Splunk API, user:pass",
+        dest="splunk_creds",
+        required=True,
+    )
+    parser.add_argument(
+        "-f",
+        "--criteria-filter",
+        help="Filter Splunk search results criteria field. Default: *",
+        dest="criteria_filter",
+        required=False,
+        default="*",
+    )
+    parser.add_argument(
+        "-c",
+        "--csv-report",
+        help="Splunk csv file from which to pull data.",
+        required=True,
+        dest="csv_report",
+    )
+    parser.add_argument(
+        "--csv-key",
+        help="Key(s) to apply to config from the csv. If not specified, applies all supported keys.",
+        choices=SUPPORTED_CSV_KEYS,
+        required=False,
+        nargs="*",
+        default=None,
+        dest="csv_keys",
+    )
+    parser.add_argument(
+        "--app",
+        help="Splunk app of the CSV file",
+        default="yelp_computeinfra",
+        required=False,
+        dest="splunk_app",
+    )
+    parser.add_argument(
+        "--git-remote",
+        help="Master git repo for soaconfigs",
+        default=None,
+        dest="git_remote",
+    )
+    parser.add_argument(
+        "--branch",
+        help="Branch name to push to. Defaults to master",
+        default="master",
+        required=False,
+        dest="branch",
+    )
+    parser.add_argument(
+        "--push-to-remote",
+        help="Actually push to remote. Otherwise files will only be modified and validated.",
+        action="store_true",
+        dest="push_to_remote",
+    )
+    parser.add_argument(
+        "--local-dir",
+        help="Act on configs in the local directory rather than cloning the git_remote",
+        required=False,
+        default=None,
+        dest="local_dir",
+    )
+    parser.add_argument(
+        "--source-id",
+        help="String to attribute the changes in the commit message. Defaults to csv report name",
+        required=False,
+        default=None,
+        dest="source_id",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        help="Logging verbosity",
+        action="store_true",
+        dest="verbose",
+    )
+    parser.add_argument(
+        "--exclude-clusters",
+        required=False,
+        default=None,
+        nargs="+",
+    )
+    return parser.parse_args()
+
+
+def get_default_git_remote():
+    system_paasta_config = load_system_paasta_config()
+    repo_config = system_paasta_config.get_git_repo_config("yelpsoa-configs")
+    default_git_remote = format_git_url(
+        system_paasta_config.get_git_config()["git_user"],
+        repo_config.get("git_server", DEFAULT_SOA_CONFIGS_GIT_URL),
+        repo_config["repo_name"],
+    )
+    return default_git_remote
+
+
+SupportedInstanceType = Literal["kubernetes", "eks", "cassandracluster"]
+
+
+class CassandraRightsizerResult(TypedDict):
+    current_cpus: str
+    suggested_cpus: str
+
+    current_disk: str
+    suggested_disk: str
+
+    current_mem: str
+    suggested_mem: str
+
+    current_replicas: str
+    suggested_replicas: str
+
+
+class CassandraRecommendation(TypedDict, total=False):
+    disk: str
+    mem: str
+    cpus: float
+    replicas: int
+    cpu_burst_percent: float
+
+
+class KubernetesRightsizerResult(TypedDict):
+    current_cpus: str
+    suggested_cpus: str
+
+    current_disk: str
+    suggested_disk: str
+
+    current_mem: str
+    suggested_mem: str
+
+    suggested_hacheck_cpus: float
+
+    suggested_cpu_burst_add: float
+
+    suggested_min_instances: int
+
+    suggested_max_instances: int
+
+
+class KubernetesRecommendation(TypedDict, total=False):
+    disk: float
+    mem: float
+    cpus: float
+    cpu_burst_add: float
+    max_instances: int
+    min_instances: int
+    sidecar_resource_requirements: Dict[str, SidecarResourceRequirements]
+
+
+def get_kubernetes_recommendation_from_result(
+    result: KubernetesRightsizerResult, keys_to_apply: List[str]
+) -> KubernetesRecommendation:
+    rec: KubernetesRecommendation = {}
+    for key in keys_to_apply:
+        val: Optional[str] = cast(Optional[str], result.get(key))
+        if not val or val == NULL:
+            continue
+        if key == "cpus":
+            rec["cpus"] = float(val)
+        elif key == "cpu_burst_add":
+            rec["cpu_burst_add"] = min(1, float(val))
+        elif key == "mem":
+            rec["mem"] = max(128, round(float(val)))
+        elif key == "disk":
+            rec["disk"] = max(128, round(float(val)))
+        elif key == "min_instances":
+            rec["min_instances"] = int(val)
+        elif key == "max_instances":
+            rec["max_instances"] = int(val)
+        elif key == "hacheck_cpus":
+            hacheck_cpus_value = max(0.1, min(float(val), 1))
+            rec["sidecar_resource_requirements"] = {
+                "hacheck": {
+                    "requests": {
+                        "cpu": hacheck_cpus_value,
+                    },
+                    "limits": {
+                        "cpu": hacheck_cpus_value,
+                    },
+                },
+            }
+    return rec
+
+
+def get_cassandra_recommendation_from_result(
+    result: CassandraRightsizerResult, keys_to_apply: List[str]
+) -> CassandraRecommendation:
+    rec: CassandraRecommendation = {}
+    for key in keys_to_apply:
+        val: Optional[str] = cast(Optional[str], result.get(key))
+        if not val or val == NULL:
+            continue
+        if key == "cpus":
+            rec["cpus"] = float(val)
+        elif key == "cpu_burst_percent":
+            rec["cpu_burst_percent"] = float(val)
+        elif key == "mem":
+            rec["mem"] = val
+        elif key == "disk":
+            rec["disk"] = val
+        elif key == "replicas":
+            rec["replicas"] = int(val)
+    return rec
+
+
+def get_recommendations_by_service_file(
+    results,
+    keys_to_apply,
+    exclude_clusters: Set[str],
+):
+    results_by_service_file: Dict[tuple, Dict[str, Any]] = defaultdict(dict)
+    for result in results.values():
+        # we occasionally want to disable autotune for a cluster (or set of clusters)
+        # to do so, we can simply skip getting recommendations for any (service, cluster)
+        # pairing that includes the cluster(s) to disable
+        if result["cluster"] in exclude_clusters:
+            print(
+                f"{result['service']}.{result['instance']} in {result['cluster']} skipped due to disabled cluster."
+            )
+            continue
+
+        key = (
+            result["service"],
+            result["cluster"],
+        )  # e.g. (foo, kubernetes-norcal-stagef)
+        instance_type = result["cluster"].split("-", 1)[0]
+        rec: Union[KubernetesRecommendation, CassandraRecommendation] = {}
+        if instance_type == "cassandracluster":
+            rec = get_cassandra_recommendation_from_result(result, keys_to_apply)
+        elif instance_type == "kubernetes":
+            rec = get_kubernetes_recommendation_from_result(result, keys_to_apply)
+        if not rec:
+            continue
+        results_by_service_file[key][result["instance"]] = rec
+    return results_by_service_file
+
+
+def get_extra_message(splunk_search_string):
+    return f"""Updated {AUTO_SOACONFIG_SUBDIR}. This review is based on results from the following Splunk search:\n
+    {splunk_search_string}
+    """
+
+
+def main(args):
+    report = get_report_from_splunk(
+        args.splunk_creds, args.splunk_app, args.csv_report, args.criteria_filter
+    )
+    extra_message = get_extra_message(report["search"])
+    config_source = args.source_id or args.csv_report
+
+    keys_to_apply = args.csv_keys or SUPPORTED_CSV_KEYS
+    results = get_recommendations_by_service_file(
+        report["results"],
+        keys_to_apply,
+        exclude_clusters={
+            f"kubernetes-{cluster}" for cluster in (args.exclude_clusters or [])
+        },
+    )
+    updater = AutoConfigUpdater(
+        config_source=config_source,
+        git_remote=args.git_remote or get_default_git_remote(),
+        branch=args.branch,
+        working_dir=args.local_dir or "/nail/tmp",
+        do_clone=args.local_dir is None,
+        validation_schema_path=AUTO_SOACONFIG_SUBDIR,
+    )
+    with updater:
+        for (
+            service,
+            instance_type_cluster,
+        ), instance_recommendations in updater.merge_recommendations(results).items():
+            log.info(
+                f"Writing configs for {service} to {AUTO_SOACONFIG_SUBDIR}/{instance_type_cluster}.yaml..."
+            )
+            updater.write_configs(
+                service,
+                instance_type_cluster,
+                instance_recommendations,
+                AUTO_SOACONFIG_SUBDIR,
+                HEADER_COMMENT,
+            )
+
+        if args.push_to_remote:
+            updater.commit_to_remote(extra_message=extra_message)
+        else:
+            updater.validate()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+    main(args)

paasta_tools/contrib/service_shard_remove.py

@@ -0,0 +1,157 @@
+import argparse
+import logging
+
+from paasta_tools.config_utils import AutoConfigUpdater
+from paasta_tools.utils import DEFAULT_SOA_CONFIGS_GIT_URL
+from paasta_tools.utils import format_git_url
+from paasta_tools.utils import load_system_paasta_config
+
+log = logging.getLogger(__name__)
+
+
+def get_default_git_remote() -> str:
+    system_paasta_config = load_system_paasta_config()
+    repo_config = system_paasta_config.get_git_repo_config("yelpsoa-configs")
+    default_git_remote = format_git_url(
+        system_paasta_config.get_git_config()["git_user"],
+        repo_config.get("git_server", DEFAULT_SOA_CONFIGS_GIT_URL),
+        repo_config["repo_name"],
+    )
+    return default_git_remote
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="")
+    parser.add_argument(
+        "--git-remote",
+        help="Master git repo for soaconfigs",
+        default=get_default_git_remote(),
+        dest="git_remote",
+    )
+    parser.add_argument(
+        "--branch",
+        help="Branch name to push to",
+        required=True,
+        dest="branch",
+    )
+    parser.add_argument(
+        "--local-dir",
+        help="Act on configs in the local directory rather than cloning the git_remote",
+        required=False,
+        default=None,
+        dest="local_dir",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        help="Logging verbosity",
+        action="store_true",
+        dest="verbose",
+    )
+    parser.add_argument(
+        "--source-id",
+        help="String to attribute the changes in the commit message.",
+        required=False,
+        default=None,
+        dest="source_id",
+    )
+    parser.add_argument(
+        "--service",
+        help="Service to modify",
+        required=True,
+        dest="service",
+    )
+    parser.add_argument(
+        "--shard-name",
+        help="Shard name to remove",
+        required=True,
+        dest="shard_name",
+    )
+    parser.add_argument(
+        "--superregion",
+        nargs="+",
+        help="Superregion to remove shard from",
+        required=True,
+    )
+    return parser.parse_args()
+
+
+SUPPERREGION_DEPLOY_MAPPINGS = {
+    "norcal-devc": "dev",
+    "norcal-stagef": "stage",
+    "norcal-stageg": "stage",
+    "nova-prod": "prod",
+    "pnw-prod": "prod",
+}
+
+
+def main(args):
+    updater = AutoConfigUpdater(
+        config_source=args.source_id,
+        git_remote=args.git_remote,
+        branch=args.branch,
+        working_dir=args.local_dir or "/nail/tmp",
+        do_clone=args.local_dir is None,
+    )
+    with updater:
+        deploy_file = updater.get_existing_configs(args.service, "deploy")
+        smartstack_file = updater.get_existing_configs(args.service, "smartstack")
+        delete_all = set(args.superregion) == set(SUPPERREGION_DEPLOY_MAPPINGS.keys())
+
+        for superregion in args.superregion:
+            # Determine / load configuration (PAASTA-18216)
+            eks_config = updater.get_existing_configs(
+                args.service, f"eks-{superregion}"
+            )
+            kube_config = updater.get_existing_configs(
+                args.service, f"kubernetes-{superregion}"
+            )
+
+            if eks_config:
+                config_file = eks_config
+                config_file_suffix = "eks"
+            elif kube_config:
+                config_file = kube_config
+                config_file_suffix = "kubernetes"
+            else:
+                log.error(f"No EKS or Kubernetes config found for {args.service}")
+                continue
+
+            # Remove shard from deploy pipeline config
+            targeted_step = (
+                f"{SUPPERREGION_DEPLOY_MAPPINGS[superregion]}.{args.shard_name}"
+            )
+            deploy_file["pipeline"] = [
+                i for i in deploy_file["pipeline"] if i["step"] != targeted_step
+            ]
+            log.info(f"{targeted_step} removed from deploy config")
+            updater.write_configs(args.service, "deploy", deploy_file)
+
+            # Remove shard from corresponding config
+            del config_file[args.shard_name]
+            updater.write_configs(
+                args.service, f"{config_file_suffix}-{superregion}", config_file
+            )
+            log.info(
+                f"{args.shard_name} removed from {config_file_suffix}-{superregion}"
+            )
+
+        # If we are removing the shard from all regions, remove it from smartstack too.
+        if delete_all:
+            del smartstack_file[args.shard_name]
+            updater.write_configs(args.service, "smartstack", smartstack_file)
+            log.info(f"{args.shard_name} removed from smartstack")
+        else:
+            log.info(f"not removing {args.shard_name} from smartstack")
+
+        updater.commit_to_remote()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.WARNING)
+
+    main(args)