paasta-tools 1.21.3__py3-none-any.whl

paasta_tools/check_cassandracluster_services_replication.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# Copyright 2015-2019 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage: ./check_cassandracluster_services_replication.py [options]
+"""
+import logging
+
+from paasta_tools import cassandracluster_tools
+from paasta_tools.check_kubernetes_services_replication import (
+    check_kubernetes_pod_replication,
+)
+from paasta_tools.check_services_replication_tools import main
+
+
+log = logging.getLogger(__name__)
+
+
+if __name__ == "__main__":
+    main(
+        cassandracluster_tools.CassandraClusterDeploymentConfig,
+        check_kubernetes_pod_replication,
+        namespace="paasta-cassandraclusters",
+    )

paasta_tools/check_flink_services_health.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python
+# Copyright 2015-2019 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage: ./check_flink_services_health.py [options]
+"""
+import datetime
+import logging
+from typing import Dict
+from typing import List
+from typing import Sequence
+from typing import Tuple
+
+import pysensu_yelp
+
+from paasta_tools import flink_tools
+from paasta_tools import flinkeks_tools
+from paasta_tools.check_services_replication_tools import main
+from paasta_tools.check_services_replication_tools import parse_args
+from paasta_tools.flink_tools import FlinkDeploymentConfig
+from paasta_tools.kubernetes_tools import is_pod_ready
+from paasta_tools.kubernetes_tools import V1Pod
+from paasta_tools.monitoring_tools import check_under_replication
+from paasta_tools.monitoring_tools import send_replication_event
+from paasta_tools.smartstack_tools import KubeSmartstackEnvoyReplicationChecker
+from paasta_tools.utils import is_under_replicated
+
+
+log = logging.getLogger(__name__)
+
+
+def container_lifetime(
+    pod: V1Pod,
+) -> datetime.timedelta:
+    """Return a time duration for how long the pod is alive"""
+    st = pod.status.start_time
+    return datetime.datetime.now(st.tzinfo) - st
+
+
+def healthy_flink_containers_cnt(si_pods: Sequence[V1Pod], container_type: str) -> int:
+    """Return count of healthy Flink containers with given type"""
+    return len(
+        [
+            pod
+            for pod in si_pods
+            if pod.metadata.labels["flink.yelp.com/container-type"] == container_type
+            and is_pod_ready(pod)
+            and container_lifetime(pod).total_seconds() > 60
+        ]
+    )
+
+
+def check_under_registered_taskmanagers(
+    instance_config: FlinkDeploymentConfig,
+    expected_count: int,
+    cr_name: str,
+    is_eks: bool,
+) -> Tuple[bool, str, str]:
+    """Check if not enough taskmanagers have been registered to the jobmanager and
+    returns both the result of the check in the form of a boolean and a human-readable
+    text to be used in logging or monitoring events.
+    """
+    unhealthy = True
+    if cr_name != "":
+        try:
+            overview = flink_tools.get_flink_jobmanager_overview(
+                cr_name, instance_config.cluster, is_eks
+            )
+            num_reported = overview.get("taskmanagers", 0)
+            crit_threshold = instance_config.get_replication_crit_percentage()
+            output = (
+                f"{instance_config.job_id} has {num_reported}/{expected_count} "
+                f"taskmanagers reported by dashboard (threshold: {crit_threshold}%)"
+            )
+            unhealthy, _ = is_under_replicated(
+                num_reported, expected_count, crit_threshold
+            )
+        except ValueError as e:
+            output = (
+                f"Dashboard of service {instance_config.job_id} is not available ({e})"
+            )
+    else:
+        output = f"Dashboard of service {instance_config.job_id} is not available"
+    if unhealthy:
+        description = f"""
+This alert means that the Flink dashboard is not reporting the expected
+number of taskmanagers.
+
+Reasons this might be happening:
+
+  The service may simply be unhealthy. There also may not be enough resources
+  in the cluster to support the requested instance count.
+
+Things you can do:
+
+  * Fix the cause of the unhealthy service. Try running:
+
+     paasta status -s {instance_config.service} -i {instance_config.instance} -c {instance_config.cluster} -vv
+
+"""
+    else:
+        description = f"{instance_config.job_id} taskmanager is available"
+    return unhealthy, output, description
+
+
+def get_cr_name(si_pods: Sequence[V1Pod]) -> str:
+    """Returns the flink custom resource name based on the pod name.  We are randomly choosing jobmanager pod here.
+    This change is related to FLINK-3129
+    """
+    jobmanager_pod = [
+        pod
+        for pod in si_pods
+        if pod.metadata.labels["flink.yelp.com/container-type"] == "jobmanager"
+        and is_pod_ready(pod)
+        and container_lifetime(pod).total_seconds() > 60
+    ]
+    if len(jobmanager_pod) == 1:
+        return jobmanager_pod[0].metadata.name.split("-jobmanager-")[0]
+    else:
+        return ""
+
+
+def check_flink_service_health(
+    instance_config: FlinkDeploymentConfig,
+    pods_by_service_instance: Dict[str, Dict[str, List[V1Pod]]],
+    replication_checker: KubeSmartstackEnvoyReplicationChecker,
+    dry_run: bool = False,
+) -> None:
+    si_pods = pods_by_service_instance.get(instance_config.service, {}).get(
+        instance_config.instance, []
+    )
+    taskmanagers_expected_cnt = instance_config.config_dict.get(
+        "taskmanager", {"instances": 10}
+    ).get("instances", 10)
+    num_healthy_supervisors = healthy_flink_containers_cnt(si_pods, "supervisor")
+    num_healthy_jobmanagers = healthy_flink_containers_cnt(si_pods, "jobmanager")
+    num_healthy_taskmanagers = healthy_flink_containers_cnt(si_pods, "taskmanager")
+
+    service_cr_name = get_cr_name(si_pods)
+
+    results = [
+        check_under_replication(
+            instance_config=instance_config,
+            expected_count=1,
+            num_available=num_healthy_supervisors,
+            sub_component="supervisor",
+        ),
+        check_under_replication(
+            instance_config=instance_config,
+            expected_count=1,
+            num_available=num_healthy_jobmanagers,
+            sub_component="jobmanager",
+        ),
+        check_under_replication(
+            instance_config=instance_config,
+            expected_count=taskmanagers_expected_cnt,
+            num_available=num_healthy_taskmanagers,
+            sub_component="taskmanager",
+        ),
+        check_under_registered_taskmanagers(
+            instance_config=instance_config,
+            expected_count=taskmanagers_expected_cnt,
+            cr_name=service_cr_name,
+            is_eks=isinstance(instance_config, flinkeks_tools.FlinkEksDeploymentConfig),
+        ),
+    ]
+    output = ", ".join([r[1] for r in results])
+    description = "\n########\n".join([r[2] for r in results])
+    if any(r[0] for r in results):
+        log.error(output)
+        status = pysensu_yelp.Status.CRITICAL
+    else:
+        log.info(output)
+        status = pysensu_yelp.Status.OK
+    send_replication_event(
+        instance_config=instance_config,
+        status=status,
+        output=output,
+        description=description,
+        dry_run=dry_run,
+    )
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    main(
+        instance_type_class=flinkeks_tools.FlinkEksDeploymentConfig
+        if args.eks
+        else flink_tools.FlinkDeploymentConfig,
+        check_service_replication=check_flink_service_health,
+        namespace="paasta-flinks",
+    )

paasta_tools/check_kubernetes_api.py

@@ -0,0 +1,57 @@
+#!/opt/venvs/paasta-tools/bin/python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage: ./check_kubernetes_api.py [options]
+
+This is a script that checks connectivity and credentials for Kubernetes API.
+"""
+import argparse
+import logging
+import sys
+
+from paasta_tools.kubernetes_tools import KubeClient
+
+
+log = logging.getLogger(__name__)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", dest="verbose", default=False
+    )
+    options = parser.parse_args()
+    return options
+
+
+def main() -> None:
+    args = parse_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.WARNING)
+
+    kube_client = KubeClient()
+    try:
+        kube_client.core.list_namespace()
+        log.info("API is ok")
+        sys.exit(0)
+    except Exception as exc:
+        log.error(f"Error connecting to API: {exc}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

paasta_tools/check_kubernetes_services_replication.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+# Copyright 2015-2019 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage: ./check_kubernetes_services_replication.py [options]
+
+This is a script that checks the number of HAProxy backends via Synapse against
+the expected amount that should've been deployed via Kubernetes.
+
+Basically, the script checks smartstack.yaml for listed namespaces, and then queries
+Synapse for the number of available backends for that namespace. It then goes through
+the Kubernetes service configuration file for that cluster, and sees how many instances
+are expected to be available for that namespace based on the number of instances deployed
+on that namespace.
+
+After retrieving that information, a fraction of available instances is calculated
+(available/expected), and then compared against a threshold. The default threshold is
+50, meaning if less than 50% of a service's backends are available, the script sends
+CRITICAL. If replication_threshold is defined in the yelpsoa config for a service
+instance then it will be used instead.
+"""
+import logging
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Union
+
+from paasta_tools import eks_tools
+from paasta_tools import kubernetes_tools
+from paasta_tools import monitoring_tools
+from paasta_tools.check_services_replication_tools import main
+from paasta_tools.check_services_replication_tools import parse_args
+from paasta_tools.eks_tools import EksDeploymentConfig
+from paasta_tools.kubernetes_tools import is_pod_ready
+from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig
+from paasta_tools.kubernetes_tools import V1Pod
+from paasta_tools.long_running_service_tools import get_proxy_port_for_instance
+from paasta_tools.smartstack_tools import KubeSmartstackEnvoyReplicationChecker
+
+
+log = logging.getLogger(__name__)
+DEFAULT_ALERT_AFTER = "10m"
+
+
+def check_healthy_kubernetes_tasks_for_service_instance(
+    instance_config: Union[KubernetesDeploymentConfig, EksDeploymentConfig],
+    expected_count: int,
+    pods_by_service_instance: Dict[str, Dict[str, List[V1Pod]]],
+    dry_run: bool = False,
+) -> None:
+    si_pods = pods_by_service_instance.get(instance_config.service, {}).get(
+        instance_config.instance, []
+    )
+
+    num_healthy_tasks = len([pod for pod in si_pods if is_pod_ready(pod)])
+    log.info(
+        f"Checking {instance_config.service}.{instance_config.instance} in kubernetes as it is not in smartstack"
+    )
+    monitoring_tools.send_replication_event_if_under_replication(
+        instance_config=instance_config,
+        expected_count=expected_count,
+        num_available=num_healthy_tasks,
+        dry_run=dry_run,
+    )
+
+
+def check_kubernetes_pod_replication(
+    instance_config: Union[KubernetesDeploymentConfig, EksDeploymentConfig],
+    pods_by_service_instance: Dict[str, Dict[str, List[V1Pod]]],
+    replication_checker: KubeSmartstackEnvoyReplicationChecker,
+    dry_run: bool = False,
+) -> Optional[bool]:
+    """Checks a service's replication levels based on how the service's replication
+    should be monitored. (smartstack/envoy or k8s)
+
+    :param instance_config: an instance of KubernetesDeploymentConfig or EksDeploymentConfig
+    :param replication_checker: an instance of KubeSmartstackEnvoyReplicationChecker
+    """
+    default_alert_after = DEFAULT_ALERT_AFTER
+    expected_count = instance_config.get_instances()
+    log.info(
+        "Expecting %d total tasks for %s" % (expected_count, instance_config.job_id)
+    )
+    proxy_port = get_proxy_port_for_instance(instance_config)
+
+    registrations = instance_config.get_registrations()
+
+    # If this instance does not autoscale and only has 1 instance, set alert after to 20m.
+    # Otherwise, set it to 10 min.
+    if (
+        not instance_config.is_autoscaling_enabled()
+        and instance_config.get_instances() == 1
+    ):
+        default_alert_after = "20m"
+    if "monitoring" not in instance_config.config_dict:
+        instance_config.config_dict["monitoring"] = {}
+    instance_config.config_dict["monitoring"][
+        "alert_after"
+    ] = instance_config.config_dict["monitoring"].get(
+        "alert_after", default_alert_after
+    )
+
+    # if the primary registration does not match the service_instance name then
+    # the best we can do is check k8s for replication (for now).
+    if proxy_port is not None and registrations[0] == instance_config.job_id:
+        is_well_replicated = monitoring_tools.check_replication_for_instance(
+            instance_config=instance_config,
+            expected_count=expected_count,
+            replication_checker=replication_checker,
+            dry_run=dry_run,
+        )
+        return is_well_replicated
+    else:
+        check_healthy_kubernetes_tasks_for_service_instance(
+            instance_config=instance_config,
+            expected_count=expected_count,
+            pods_by_service_instance=pods_by_service_instance,
+            dry_run=dry_run,
+        )
+        return None
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    main(
+        instance_type_class=eks_tools.EksDeploymentConfig
+        if args.eks
+        else kubernetes_tools.KubernetesDeploymentConfig,
+        check_service_replication=check_kubernetes_pod_replication,
+    )

paasta_tools/check_oom_events.py

@@ -0,0 +1,244 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import json
+import sys
+import time
+
+from pysensu_yelp import Status
+
+from paasta_tools import monitoring_tools
+from paasta_tools.cli.cmds.logs import scribe_env_to_locations
+from paasta_tools.cli.utils import get_instance_config
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import get_services_for_cluster
+from paasta_tools.utils import load_system_paasta_config
+
+try:
+    from scribereader import scribereader
+    from scribereader.clog.readers import StreamTailerSetupError
+except ImportError:
+    scribereader = None
+
+
+OOM_EVENTS_STREAM = "tmp_paasta_oom_events"
+
+
+def compose_check_name_for_service_instance(check_name, service, instance):
+    return f"{check_name}.{service}.{instance}"
+
+
+def parse_args(args):
+    parser = argparse.ArgumentParser(
+        description=(
+            "Check the %s stream and report to Sensu if"
+            " there are any OOM events." % OOM_EVENTS_STREAM
+        )
+    )
+    parser.add_argument(
+        "-d",
+        "--soa-dir",
+        dest="soa_dir",
+        default=DEFAULT_SOA_DIR,
+        help="define a different soa config directory",
+    )
+    parser.add_argument(
+        "-r",
+        "--realert-every",
+        dest="realert_every",
+        type=int,
+        default=1,
+        help="Sensu 'realert_every' to use.",
+    )
+    parser.add_argument(
+        "--check-interval",
+        dest="check_interval",
+        type=int,
+        default=1,
+        help="How often this check runs, in minutes.",
+    )
+    parser.add_argument(
+        "--alert-threshold",
+        dest="alert_threshold",
+        type=int,
+        default=1,
+        help="Number of OOM kills required in the check interval to send an alert.",
+    )
+    parser.add_argument(
+        "-s",
+        "--superregion",
+        dest="superregion",
+        required=True,
+        help="The superregion to read OOM events from.",
+    )
+    parser.add_argument(
+        "--dry-run",
+        dest="dry_run",
+        action="store_true",
+        help="Print Sensu alert events instead of sending them",
+    )
+    return parser.parse_args(args)
+
+
+def read_oom_events_from_scribe(cluster, superregion, num_lines=1000):
+    """Read the latest 'num_lines' lines from OOM_EVENTS_STREAM and iterate over them."""
+    # paasta configs incls a map for cluster -> env that is expected by scribe
+    log_reader_config = load_system_paasta_config().get_log_reader()
+    cluster_map = log_reader_config["options"]["cluster_map"]
+    scribe_env = cluster_map[cluster]
+
+    # `scribe_env_to_locations` slightly mutates the scribe env based on whether
+    # or not it is in dev or prod
+    host, port = scribereader.get_tail_host_and_port(
+        **scribe_env_to_locations(scribe_env),
+    )
+    stream = scribereader.get_stream_tailer(
+        stream_name=OOM_EVENTS_STREAM,
+        tailing_host=host,
+        tailing_port=port,
+        lines=num_lines,
+        superregion=superregion,
+    )
+    try:
+        for line in stream:
+            try:
+                j = json.loads(line)
+                if j.get("cluster", "") == cluster:
+                    yield j
+            except json.decoder.JSONDecodeError:
+                pass
+    except StreamTailerSetupError as e:
+        if "No data in stream" in str(e):
+            pass
+        else:
+            raise e
+
+
+def latest_oom_events(cluster, superregion, interval=60):
+    """
+    :returns: {(service, instance): [OOMEvent, OOMEvent,...] }
+              if the number of events > 0
+    """
+    start_timestamp = int(time.time()) - interval
+    res = {}
+    for e in read_oom_events_from_scribe(cluster, superregion):
+        if e["timestamp"] > start_timestamp:
+            key = (e["service"], e["instance"])
+            res.setdefault(key, set()).add(e.get("container_id", ""))
+    return res
+
+
+def compose_sensu_status(
+    instance, oom_events, is_check_enabled, alert_threshold, check_interval
+):
+    """
+    :param instance: InstanceConfig
+    :param oom_events: a list of OOMEvents
+    :param is_check_enabled: boolean to indicate whether the check enabled for the instance
+    """
+    interval_string = f"{check_interval} minute(s)"
+    instance_name = f"{instance.service}.{instance.instance}"
+    if not is_check_enabled:
+        return (Status.OK, f"This check is disabled for {instance_name}.")
+    if not oom_events:
+        return (
+            Status.OK,
+            f"No oom events for {instance_name} in the last {interval_string}.",
+        )
+    elif len(oom_events) >= alert_threshold:
+        return (
+            Status.CRITICAL,
+            f"The Out Of Memory killer killed processes for {instance_name} "
+            f"in the last {interval_string}.",
+        )
+    else:
+        # If the number of OOM kills isn't above the alert threshold,
+        # don't send anything. This will keep an alert open if it's already open,
+        # but won't start a new alert if there wasn't one yet
+        return None
+
+
+def send_sensu_event(instance, oom_events, args):
+    """
+    :param instance: InstanceConfig
+    :param oom_events: a list of OOMEvents
+    """
+    check_name = compose_check_name_for_service_instance(
+        "oom-killer", instance.service, instance.instance
+    )
+    monitoring_overrides = instance.get_monitoring()
+    status = compose_sensu_status(
+        instance=instance,
+        oom_events=oom_events,
+        is_check_enabled=monitoring_overrides.get("check_oom_events", True),
+        alert_threshold=args.alert_threshold,
+        check_interval=args.check_interval,
+    )
+    if not status:
+        return
+
+    memory_limit = instance.get_mem()
+    try:
+        memory_limit_str = f"{int(memory_limit)}MB"
+    except ValueError:
+        memory_limit_str = memory_limit
+
+    monitoring_overrides.update(
+        {
+            "page": False,
+            "alert_after": "0m",
+            "realert_every": args.realert_every,
+            "runbook": "y/check-oom-events",
+            "tip": (
+                "Follow the runbook to investigate and rightsize memory usage "
+                f"(curr: {memory_limit_str})"
+            ),
+        }
+    )
+    return monitoring_tools.send_event(
+        service=instance.service,
+        check_name=check_name,
+        overrides=monitoring_overrides,
+        status=status[0],
+        output=status[1],
+        soa_dir=instance.soa_dir,
+        dry_run=args.dry_run,
+    )
+
+
+def main(sys_argv):
+    args = parse_args(sys_argv[1:])
+    cluster = load_system_paasta_config().get_cluster()
+    victims = latest_oom_events(
+        cluster, args.superregion, interval=(60 * args.check_interval)
+    )
+
+    for (service, instance) in get_services_for_cluster(cluster, soa_dir=args.soa_dir):
+        try:
+            instance_config = get_instance_config(
+                service=service,
+                instance=instance,
+                cluster=cluster,
+                load_deployments=False,
+                soa_dir=args.soa_dir,
+            )
+            oom_events = victims.get((service, instance), [])
+            send_sensu_event(instance_config, oom_events, args)
+        except NotImplementedError:  # When instance_type is not supported by get_instance_config
+            pass
+
+
+if __name__ == "__main__":
+    main(sys.argv)