paasta-tools 1.21.3__py3-none-any.whl

paasta_tools/yaml_tools.py

@@ -0,0 +1,44 @@
+import sys
+
+import yaml
+
+# try and catch both /opt/venvs/paasta-tools and ~/pg/paasta/.tox/py38-linux as if we're being run as an application,
+# we likely want to fail on a potential slowdown rather than experience a performance regression
+if "paasta" in sys.prefix:
+    from yaml import CSafeLoader as Loader
+    from yaml import CSafeDumper as Dumper
+# but for the vanishingly few instances where folks add us as as dependency, we don't enforce that they use the
+# C-accelerated Loader/Dumper
+else:
+    try:
+        from yaml import CSafeLoader as Loader
+        from yaml import CSafeDumper as Dumper
+    except ImportError:  # pragma: no cover
+        from yaml import SafeLoader as Loader  # type: ignore
+        from yaml import SafeDumper as Dumper  # type: ignore
+
+
+def dump(*args, **kwargs):
+    kwargs["Dumper"] = Dumper
+    return yaml.dump(*args, **kwargs)
+
+
+def dump_all(*args, **kwargs):
+    kwargs["Dumper"] = Dumper
+    return yaml.dump_all(*args, **kwargs)
+
+
+def load(*args, **kwargs):
+    kwargs["Loader"] = Loader
+    return yaml.load(*args, **kwargs)
+
+
+def load_all(*args, **kwargs):
+    kwargs["Loader"] = Loader
+    return yaml.load_all(*args, **kwargs)
+
+
+safe_dump = dump
+safe_dump_all = dump_all
+safe_load = load
+safe_load_all = load_all

paasta_tools-1.21.3.data/scripts/apply_external_resources.py

@@ -0,0 +1,79 @@
+#!python
+import os
+import sys
+from filecmp import cmp
+from shutil import copy
+from subprocess import CalledProcessError
+from subprocess import run
+from traceback import print_exc
+
+APPLIED_DIRECTORY = ".applied"
+
+
+# This script expects the KUBECONFIG environment variable to be set correctly
+def main(puppet_resource_root: str) -> int:
+    exit_code = 0
+    applied_resource_root = os.path.join(puppet_resource_root, APPLIED_DIRECTORY)
+
+    # Loop through everything in the puppet resource path
+    for root, dirs, files in os.walk(puppet_resource_root):
+        # modifying the 'dirs' variable in-place will update the order that os.walk visits things
+        if APPLIED_DIRECTORY in dirs:
+            dirs.remove(APPLIED_DIRECTORY)
+        dirs.sort()  # Need to apply things in the correct order
+
+        # Check to see if there's a difference between what Puppet created and
+        # what's been previously applied
+        for filename in sorted([f for f in files if f.endswith(".yaml")]):
+            path = os.path.join(root, filename)
+            applied_path = os.path.join(
+                applied_resource_root, os.path.relpath(path, puppet_resource_root)
+            )
+            print(f"comparing {path} and {applied_path}")
+            if not os.path.exists(applied_path) or not cmp(
+                path, applied_path, shallow=False
+            ):
+                # This is idempotent; if something gets out of sync and a resource gets applied
+                # a second time, kubectl just won't make any changes
+                try:
+                    run(["kubectl", "apply", "-f", path], check=True)
+                    os.makedirs(os.path.dirname(applied_path), exist_ok=True)
+                    copy(path, applied_path)
+                except CalledProcessError:
+                    print(f"There was a problem applying {path}:\n")
+                    print_exc(
+                        file=sys.stdout
+                    )  # keep all messages on the same stream so they're in order
+                    exit_code = 1
+                    continue
+
+    # Loop through all the files that have been previously applied and see
+    # if Puppet has removed any of them
+    for root, dirs, files in os.walk(applied_resource_root):
+        dirs.sort(reverse=True)  # for deleting things, we need to go in reverse order
+        for filename in sorted([f for f in files if f.endswith(".yaml")], reverse=True):
+            path = os.path.join(root, filename)
+            puppet_path = os.path.join(
+                puppet_resource_root, os.path.relpath(path, applied_resource_root)
+            )
+            if not os.path.exists(puppet_path):
+                print(f"Deleting resource {path}...")
+                try:
+                    run(
+                        ["kubectl", "delete", "--ignore-not-found=true", "-f", path],
+                        check=True,
+                    )
+                    os.remove(path)
+                except CalledProcessError:
+                    print(f"There was a problem deleting {path}:\n")
+                    print_exc(
+                        file=sys.stdout
+                    )  # keep all messages on the same stream so they're in order
+                    exit_code = 1
+                    continue
+
+    return exit_code
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1]))

paasta_tools-1.21.3.data/scripts/bounce_log_latency_parser.py

@@ -0,0 +1,68 @@
+#!python
+import itertools
+import json
+import sys
+from collections import defaultdict
+from datetime import datetime
+
+
+def get_datetime_from_ts(ts):
+    tformat = "%Y-%m-%dT%H:%M:%S.%f"
+    return datetime.strptime(ts, tformat)
+
+
+def get_deploy_durations_from_file(filename):
+    """
+    filename: path to a file to be parsed for datetime data
+    The expected input is a paasta service log for the deploy events
+    The way I've been fetching them is by running 'internal logreader command' | grep deploy | grep event > filename
+    """
+    file_object = open(filename, "r")
+    data = sorted(
+        [json.loads(line.rstrip("\n")) for line in file_object],
+        key=lambda x: get_datetime_from_ts(x["timestamp"]),
+    )
+
+    timedeltas = defaultdict(list)
+    last_time = dict()
+    instance_bitvector = defaultdict(bool)  # defaults to False
+
+    for datum in data:
+        time = get_datetime_from_ts(datum["timestamp"])
+        instance = datum["instance"]
+        if "in progress" in datum["message"] and not instance_bitvector[instance]:
+            instance_bitvector[instance] = True
+            last_time[instance] = time
+        elif "finishing" in datum["message"]:
+            instance_bitvector[instance] = False
+            timedeltas[instance].append(time - last_time[instance])
+
+    return timedeltas
+
+
+def display_bounce_info(timedeltas):
+    """
+    timedeltas: iterable of timedelta objects
+    """
+    std = list(sorted(timedeltas))
+    print("Median time to bounce: {} seconds".format(std[len(std) / 2]))
+    print("10% time to bounce: {}".format(std[len(std) / 10]))
+    print("90% time to bounce: {}".format(std[len(std) * 9 / 10]))
+
+
+def main(filenames):
+    for filename in filenames:
+        print(filename)
+        print("=========================")
+        timedeltas = get_deploy_durations_from_file(filename)
+        for instance, tdlist in timedeltas.items():
+            if timedeltas:
+                print("Instance: %s" % instance)
+                display_bounce_info(tdlist)
+        print("Overall:")
+        display_bounce_info(itertools.chain.from_iterable(timedeltas.values()))
+        print("=========================")
+
+
+if __name__ == "__main__":
+    main(filenames=sys.argv[1:])

paasta_tools-1.21.3.data/scripts/check_autoscaler_max_instances.py

@@ -0,0 +1,212 @@
+#!python
+import argparse
+import asyncio
+import logging
+from typing import Type
+
+import pysensu_yelp
+
+from paasta_tools.eks_tools import EksDeploymentConfig
+from paasta_tools.instance import kubernetes as pik
+from paasta_tools.kubernetes_tools import get_kubernetes_app_name
+from paasta_tools.kubernetes_tools import KubeClient
+from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig
+from paasta_tools.metrics.metastatus_lib import suffixed_number_value
+from paasta_tools.monitoring_tools import send_event
+from paasta_tools.paasta_service_config_loader import PaastaServiceConfigLoader
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import list_services
+from paasta_tools.utils import load_system_paasta_config
+from paasta_tools.utils import SystemPaastaConfig
+
+log = logging.getLogger(__name__)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description=(
+            "Check all autoscaled services to see if they're at their max_instances. If"
+            " so, send an alert if their utilization is above"
+            " max_instances_alert_threshold."
+        )
+    )
+    parser.add_argument(
+        "-d",
+        "--soa-dir",
+        dest="soa_dir",
+        default=DEFAULT_SOA_DIR,
+        help="Use a different soa config directory",
+    )
+    parser.add_argument(
+        "--dry-run",
+        dest="dry_run",
+        action="store_true",
+        help="Print Sensu alert events instead of sending them",
+    )
+    return parser.parse_args()
+
+
+async def check_max_instances(
+    soa_dir: str,
+    cluster: str,
+    instance_type_class: Type[KubernetesDeploymentConfig],
+    system_paasta_config: SystemPaastaConfig,
+    dry_run: bool = False,
+):
+    kube_client = KubeClient()
+    for service in list_services(soa_dir=soa_dir):
+        service_config = PaastaServiceConfigLoader(service=service, soa_dir=soa_dir)
+        for job_config in service_config.instance_configs(
+            cluster=cluster, instance_type_class=instance_type_class
+        ):
+            instance = job_config.get_instance()
+            if not job_config.get_autoscaling_metric_spec(
+                name=get_kubernetes_app_name(service, instance),
+                cluster=cluster,
+                kube_client=kube_client,
+                namespace=job_config.get_namespace(),
+            ):
+                # Not an instance that uses HPA, don't check.
+                # TODO: should we send status=0 here, in case someone disables autoscaling for their service / changes
+                # to bespoke autoscaler?
+                continue
+
+            if not job_config.get_docker_image():
+                # skip services that haven't been marked for deployment yet.
+                continue
+
+            autoscaling_status = await pik.autoscaling_status(
+                kube_client=kube_client,
+                job_config=job_config,
+                namespace=job_config.get_namespace(),
+            )
+            if autoscaling_status["min_instances"] == -1:
+                log.warning(
+                    f"HPA {job_config.get_sanitised_deployment_name()} not found."
+                )
+                continue
+
+            if (
+                autoscaling_status["min_instances"]
+                == autoscaling_status["max_instances"]
+            ) and "canary" in instance:
+                status = pysensu_yelp.Status.OK
+                output = (
+                    f"Not checking {service}.{instance} as the instance name contains"
+                    ' "canary" and min_instances == max_instances.'
+                )
+            elif (
+                autoscaling_status["desired_replicas"]
+                >= autoscaling_status["max_instances"]
+            ):
+
+                metrics_provider_configs = job_config.get_autoscaling_params()[
+                    "metrics_providers"
+                ]
+
+                status = pysensu_yelp.Status.UNKNOWN
+                output = "how are there no metrics for this thing?"
+
+                # This makes an assumption that the metrics currently used by the HPA are exactly the same order (and
+                # length) as the list of metrics_providers dictionaries. This should generally be true, but between
+                # yelpsoa-configs being pushed and the HPA actually being updated it may not be true. This might cause
+                # spurious alerts, but hopefully the frequency is low. We can add some safeguards if it's a problem.
+                # (E.g. smarter matching between the status dicts and the config dicts, or bailing/not alerting if the
+                # lists aren't the same lengths.)
+                for metric, metrics_provider_config in zip(
+                    autoscaling_status["metrics"], metrics_provider_configs
+                ):
+
+                    setpoint = metrics_provider_config["setpoint"]
+                    threshold = metrics_provider_config.get(
+                        "max_instances_alert_threshold",
+                        setpoint,
+                    )
+
+                    try:
+                        current_value = suffixed_number_value(metric["current_value"])
+                        target_value = suffixed_number_value(metric["target_value"])
+                    except KeyError:
+                        # we likely couldn't find values for the current metric from autoscaling status
+                        # if this is the only metric, we will return UNKNOWN+this error
+                        # suggest fixing their autoscaling config
+                        output = f'{service}.{instance}: Service is at max_instances, and there is an error fetching your {metrics_provider_config["type"]} metric. Check your autoscaling configs or reach out to #paasta.'
+                    else:
+                        # target_value can be 100*setpoint (for cpu), 1 (for uwsgi, piscina, gunicorn,
+                        # active_requests), or setpoint (for promql).
+                        # Here we divide current_value by target_value to find the ratio of utilization to setpoint,
+                        # and then multiply by setpoint to find the actual utilization in the same units as setpoint.
+                        utilization = setpoint * current_value / target_value
+
+                        if threshold == setpoint:
+                            threshold_description = f"setpoint ({threshold})"
+                        else:
+                            threshold_description = (
+                                f"max_instances_alert_threshold ({threshold})"
+                            )
+
+                        if utilization > threshold:
+                            status = pysensu_yelp.Status.CRITICAL
+                            output = (
+                                f"{service}.{instance}: Service is at max_instances, and"
+                                f" utilization ({utilization}) is greater than"
+                                f" {threshold_description}."
+                            )
+                        else:
+                            status = pysensu_yelp.Status.OK
+                            output = (
+                                f"{service}.{instance}: Service is at max_instances, but"
+                                f" utilization ({utilization}) is less than"
+                                f" {threshold_description}."
+                            )
+            else:
+                status = pysensu_yelp.Status.OK
+                output = f"{service}.{instance} is below max_instances."
+
+            monitoring_overrides = job_config.get_monitoring()
+            monitoring_overrides.update(
+                {
+                    "page": False,  # TODO: remove this line once this alert has been deployed for a little while.
+                    "runbook": "y/check-autoscaler-max-instances",
+                    "realert_every": 60,  # The check runs once a minute, so this would realert every hour.
+                    "tip": (
+                        "The autoscaler wants to scale up to handle additional load"
+                        " because your service is overloaded, but cannot scale any"
+                        " higher because of max_instances. You may want to bump"
+                        " max_instances. To make this alert quieter, adjust"
+                        " autoscaling.metrics_providers[n].max_instances_alert_threshold in yelpsoa-configs."
+                    ),
+                }
+            )
+            send_event(
+                service,
+                check_name=f"check_autoscaler_max_instances.{service}.{instance}",
+                overrides=monitoring_overrides,
+                status=status,
+                output=output,
+                soa_dir=soa_dir,
+                ttl=None,
+                cluster=cluster,
+                system_paasta_config=system_paasta_config,
+                dry_run=dry_run,
+            )
+
+
+def main():
+    args = parse_args()
+    system_paasta_config = load_system_paasta_config()
+
+    for instance_type_class in [KubernetesDeploymentConfig, EksDeploymentConfig]:
+        asyncio.run(
+            check_max_instances(
+                soa_dir=args.soa_dir,
+                cluster=system_paasta_config.get_cluster(),
+                instance_type_class=instance_type_class,
+                system_paasta_config=system_paasta_config,
+                dry_run=args.dry_run,
+            )
+        )
+
+
+if __name__ == "__main__":
+    main()

paasta_tools-1.21.3.data/scripts/check_cassandracluster_services_replication.py

@@ -0,0 +1,35 @@
+#!python
+# Copyright 2015-2019 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage: ./check_cassandracluster_services_replication.py [options]
+"""
+import logging
+
+from paasta_tools import cassandracluster_tools
+from paasta_tools.check_kubernetes_services_replication import (
+    check_kubernetes_pod_replication,
+)
+from paasta_tools.check_services_replication_tools import main
+
+
+log = logging.getLogger(__name__)
+
+
+if __name__ == "__main__":
+    main(
+        cassandracluster_tools.CassandraClusterDeploymentConfig,
+        check_kubernetes_pod_replication,
+        namespace="paasta-cassandraclusters",
+    )

paasta_tools-1.21.3.data/scripts/check_flink_services_health.py

@@ -0,0 +1,203 @@
+#!python
+# Copyright 2015-2019 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage: ./check_flink_services_health.py [options]
+"""
+import datetime
+import logging
+from typing import Dict
+from typing import List
+from typing import Sequence
+from typing import Tuple
+
+import pysensu_yelp
+
+from paasta_tools import flink_tools
+from paasta_tools import flinkeks_tools
+from paasta_tools.check_services_replication_tools import main
+from paasta_tools.check_services_replication_tools import parse_args
+from paasta_tools.flink_tools import FlinkDeploymentConfig
+from paasta_tools.kubernetes_tools import is_pod_ready
+from paasta_tools.kubernetes_tools import V1Pod
+from paasta_tools.monitoring_tools import check_under_replication
+from paasta_tools.monitoring_tools import send_replication_event
+from paasta_tools.smartstack_tools import KubeSmartstackEnvoyReplicationChecker
+from paasta_tools.utils import is_under_replicated
+
+
+log = logging.getLogger(__name__)
+
+
+def container_lifetime(
+    pod: V1Pod,
+) -> datetime.timedelta:
+    """Return a time duration for how long the pod is alive"""
+    st = pod.status.start_time
+    return datetime.datetime.now(st.tzinfo) - st
+
+
+def healthy_flink_containers_cnt(si_pods: Sequence[V1Pod], container_type: str) -> int:
+    """Return count of healthy Flink containers with given type"""
+    return len(
+        [
+            pod
+            for pod in si_pods
+            if pod.metadata.labels["flink.yelp.com/container-type"] == container_type
+            and is_pod_ready(pod)
+            and container_lifetime(pod).total_seconds() > 60
+        ]
+    )
+
+
+def check_under_registered_taskmanagers(
+    instance_config: FlinkDeploymentConfig,
+    expected_count: int,
+    cr_name: str,
+    is_eks: bool,
+) -> Tuple[bool, str, str]:
+    """Check if not enough taskmanagers have been registered to the jobmanager and
+    returns both the result of the check in the form of a boolean and a human-readable
+    text to be used in logging or monitoring events.
+    """
+    unhealthy = True
+    if cr_name != "":
+        try:
+            overview = flink_tools.get_flink_jobmanager_overview(
+                cr_name, instance_config.cluster, is_eks
+            )
+            num_reported = overview.get("taskmanagers", 0)
+            crit_threshold = instance_config.get_replication_crit_percentage()
+            output = (
+                f"{instance_config.job_id} has {num_reported}/{expected_count} "
+                f"taskmanagers reported by dashboard (threshold: {crit_threshold}%)"
+            )
+            unhealthy, _ = is_under_replicated(
+                num_reported, expected_count, crit_threshold
+            )
+        except ValueError as e:
+            output = (
+                f"Dashboard of service {instance_config.job_id} is not available ({e})"
+            )
+    else:
+        output = f"Dashboard of service {instance_config.job_id} is not available"
+    if unhealthy:
+        description = f"""
+This alert means that the Flink dashboard is not reporting the expected
+number of taskmanagers.
+
+Reasons this might be happening:
+
+  The service may simply be unhealthy. There also may not be enough resources
+  in the cluster to support the requested instance count.
+
+Things you can do:
+
+  * Fix the cause of the unhealthy service. Try running:
+
+     paasta status -s {instance_config.service} -i {instance_config.instance} -c {instance_config.cluster} -vv
+
+"""
+    else:
+        description = f"{instance_config.job_id} taskmanager is available"
+    return unhealthy, output, description
+
+
+def get_cr_name(si_pods: Sequence[V1Pod]) -> str:
+    """Returns the flink custom resource name based on the pod name.  We are randomly choosing jobmanager pod here.
+    This change is related to FLINK-3129
+    """
+    jobmanager_pod = [
+        pod
+        for pod in si_pods
+        if pod.metadata.labels["flink.yelp.com/container-type"] == "jobmanager"
+        and is_pod_ready(pod)
+        and container_lifetime(pod).total_seconds() > 60
+    ]
+    if len(jobmanager_pod) == 1:
+        return jobmanager_pod[0].metadata.name.split("-jobmanager-")[0]
+    else:
+        return ""
+
+
+def check_flink_service_health(
+    instance_config: FlinkDeploymentConfig,
+    pods_by_service_instance: Dict[str, Dict[str, List[V1Pod]]],
+    replication_checker: KubeSmartstackEnvoyReplicationChecker,
+    dry_run: bool = False,
+) -> None:
+    si_pods = pods_by_service_instance.get(instance_config.service, {}).get(
+        instance_config.instance, []
+    )
+    taskmanagers_expected_cnt = instance_config.config_dict.get(
+        "taskmanager", {"instances": 10}
+    ).get("instances", 10)
+    num_healthy_supervisors = healthy_flink_containers_cnt(si_pods, "supervisor")
+    num_healthy_jobmanagers = healthy_flink_containers_cnt(si_pods, "jobmanager")
+    num_healthy_taskmanagers = healthy_flink_containers_cnt(si_pods, "taskmanager")
+
+    service_cr_name = get_cr_name(si_pods)
+
+    results = [
+        check_under_replication(
+            instance_config=instance_config,
+            expected_count=1,
+            num_available=num_healthy_supervisors,
+            sub_component="supervisor",
+        ),
+        check_under_replication(
+            instance_config=instance_config,
+            expected_count=1,
+            num_available=num_healthy_jobmanagers,
+            sub_component="jobmanager",
+        ),
+        check_under_replication(
+            instance_config=instance_config,
+            expected_count=taskmanagers_expected_cnt,
+            num_available=num_healthy_taskmanagers,
+            sub_component="taskmanager",
+        ),
+        check_under_registered_taskmanagers(
+            instance_config=instance_config,
+            expected_count=taskmanagers_expected_cnt,
+            cr_name=service_cr_name,
+            is_eks=isinstance(instance_config, flinkeks_tools.FlinkEksDeploymentConfig),
+        ),
+    ]
+    output = ", ".join([r[1] for r in results])
+    description = "\n########\n".join([r[2] for r in results])
+    if any(r[0] for r in results):
+        log.error(output)
+        status = pysensu_yelp.Status.CRITICAL
+    else:
+        log.info(output)
+        status = pysensu_yelp.Status.OK
+    send_replication_event(
+        instance_config=instance_config,
+        status=status,
+        output=output,
+        description=description,
+        dry_run=dry_run,
+    )
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    main(
+        instance_type_class=flinkeks_tools.FlinkEksDeploymentConfig
+        if args.eks
+        else flink_tools.FlinkDeploymentConfig,
+        check_service_replication=check_flink_service_health,
+        namespace="paasta-flinks",
+    )