paasta-tools 1.21.3__py3-none-any.whl

paasta_tools-1.21.3.data/scripts/check_kubernetes_api.py

@@ -0,0 +1,57 @@
+#!python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage: ./check_kubernetes_api.py [options]
+
+This is a script that checks connectivity and credentials for Kubernetes API.
+"""
+import argparse
+import logging
+import sys
+
+from paasta_tools.kubernetes_tools import KubeClient
+
+
+log = logging.getLogger(__name__)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", dest="verbose", default=False
+    )
+    options = parser.parse_args()
+    return options
+
+
+def main() -> None:
+    args = parse_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.WARNING)
+
+    kube_client = KubeClient()
+    try:
+        kube_client.core.list_namespace()
+        log.info("API is ok")
+        sys.exit(0)
+    except Exception as exc:
+        log.error(f"Error connecting to API: {exc}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

paasta_tools-1.21.3.data/scripts/check_kubernetes_services_replication.py

@@ -0,0 +1,141 @@
+#!python
+# Copyright 2015-2019 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage: ./check_kubernetes_services_replication.py [options]
+
+This is a script that checks the number of HAProxy backends via Synapse against
+the expected amount that should've been deployed via Kubernetes.
+
+Basically, the script checks smartstack.yaml for listed namespaces, and then queries
+Synapse for the number of available backends for that namespace. It then goes through
+the Kubernetes service configuration file for that cluster, and sees how many instances
+are expected to be available for that namespace based on the number of instances deployed
+on that namespace.
+
+After retrieving that information, a fraction of available instances is calculated
+(available/expected), and then compared against a threshold. The default threshold is
+50, meaning if less than 50% of a service's backends are available, the script sends
+CRITICAL. If replication_threshold is defined in the yelpsoa config for a service
+instance then it will be used instead.
+"""
+import logging
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Union
+
+from paasta_tools import eks_tools
+from paasta_tools import kubernetes_tools
+from paasta_tools import monitoring_tools
+from paasta_tools.check_services_replication_tools import main
+from paasta_tools.check_services_replication_tools import parse_args
+from paasta_tools.eks_tools import EksDeploymentConfig
+from paasta_tools.kubernetes_tools import is_pod_ready
+from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig
+from paasta_tools.kubernetes_tools import V1Pod
+from paasta_tools.long_running_service_tools import get_proxy_port_for_instance
+from paasta_tools.smartstack_tools import KubeSmartstackEnvoyReplicationChecker
+
+
+log = logging.getLogger(__name__)
+DEFAULT_ALERT_AFTER = "10m"
+
+
+def check_healthy_kubernetes_tasks_for_service_instance(
+    instance_config: Union[KubernetesDeploymentConfig, EksDeploymentConfig],
+    expected_count: int,
+    pods_by_service_instance: Dict[str, Dict[str, List[V1Pod]]],
+    dry_run: bool = False,
+) -> None:
+    si_pods = pods_by_service_instance.get(instance_config.service, {}).get(
+        instance_config.instance, []
+    )
+
+    num_healthy_tasks = len([pod for pod in si_pods if is_pod_ready(pod)])
+    log.info(
+        f"Checking {instance_config.service}.{instance_config.instance} in kubernetes as it is not in smartstack"
+    )
+    monitoring_tools.send_replication_event_if_under_replication(
+        instance_config=instance_config,
+        expected_count=expected_count,
+        num_available=num_healthy_tasks,
+        dry_run=dry_run,
+    )
+
+
+def check_kubernetes_pod_replication(
+    instance_config: Union[KubernetesDeploymentConfig, EksDeploymentConfig],
+    pods_by_service_instance: Dict[str, Dict[str, List[V1Pod]]],
+    replication_checker: KubeSmartstackEnvoyReplicationChecker,
+    dry_run: bool = False,
+) -> Optional[bool]:
+    """Checks a service's replication levels based on how the service's replication
+    should be monitored. (smartstack/envoy or k8s)
+
+    :param instance_config: an instance of KubernetesDeploymentConfig or EksDeploymentConfig
+    :param replication_checker: an instance of KubeSmartstackEnvoyReplicationChecker
+    """
+    default_alert_after = DEFAULT_ALERT_AFTER
+    expected_count = instance_config.get_instances()
+    log.info(
+        "Expecting %d total tasks for %s" % (expected_count, instance_config.job_id)
+    )
+    proxy_port = get_proxy_port_for_instance(instance_config)
+
+    registrations = instance_config.get_registrations()
+
+    # If this instance does not autoscale and only has 1 instance, set alert after to 20m.
+    # Otherwise, set it to 10 min.
+    if (
+        not instance_config.is_autoscaling_enabled()
+        and instance_config.get_instances() == 1
+    ):
+        default_alert_after = "20m"
+    if "monitoring" not in instance_config.config_dict:
+        instance_config.config_dict["monitoring"] = {}
+    instance_config.config_dict["monitoring"][
+        "alert_after"
+    ] = instance_config.config_dict["monitoring"].get(
+        "alert_after", default_alert_after
+    )
+
+    # if the primary registration does not match the service_instance name then
+    # the best we can do is check k8s for replication (for now).
+    if proxy_port is not None and registrations[0] == instance_config.job_id:
+        is_well_replicated = monitoring_tools.check_replication_for_instance(
+            instance_config=instance_config,
+            expected_count=expected_count,
+            replication_checker=replication_checker,
+            dry_run=dry_run,
+        )
+        return is_well_replicated
+    else:
+        check_healthy_kubernetes_tasks_for_service_instance(
+            instance_config=instance_config,
+            expected_count=expected_count,
+            pods_by_service_instance=pods_by_service_instance,
+            dry_run=dry_run,
+        )
+        return None
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    main(
+        instance_type_class=eks_tools.EksDeploymentConfig
+        if args.eks
+        else kubernetes_tools.KubernetesDeploymentConfig,
+        check_service_replication=check_kubernetes_pod_replication,
+    )

paasta_tools-1.21.3.data/scripts/check_manual_oapi_changes.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -uo pipefail
+
+diff_names=$(git diff origin/master --name-only)
+schema_touched=$(echo "$diff_names" | grep 'paasta_tools/api/api_docs/oapi.yaml')
+api_touched=$(echo "$diff_names" | grep 'paasta_tools/paastaapi')
+
+# this case will only be hit as part of `make test` or someone running pre-commit on
+# all files manually since we've chosen to only run this hook on changes to the schema
+# file
+if [ ! -z "$api_touched" -a -z "$schema_touched" ]; then
+    echo "paasta_tools/paastaapi must not be modified manually" >&2
+    echo "Please revert your changes:" >&2
+    echo -e "$api_touched" >&2
+    exit 1
+fi
+
+make openapi-codegen &>/dev/null
+diff=$(git diff --name-only | grep paasta_tools/paastaapi)
+if [ ! -z "$diff" ]; then
+    echo "paasta_tools/paastaapi codegen has a diff, either commit the changes or fix oapi.yaml:" >&2
+    echo -e "$diff" >&2
+    exit 1
+fi

paasta_tools-1.21.3.data/scripts/check_oom_events.py

@@ -0,0 +1,244 @@
+#!python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import json
+import sys
+import time
+
+from pysensu_yelp import Status
+
+from paasta_tools import monitoring_tools
+from paasta_tools.cli.cmds.logs import scribe_env_to_locations
+from paasta_tools.cli.utils import get_instance_config
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import get_services_for_cluster
+from paasta_tools.utils import load_system_paasta_config
+
+try:
+    from scribereader import scribereader
+    from scribereader.clog.readers import StreamTailerSetupError
+except ImportError:
+    scribereader = None
+
+
+OOM_EVENTS_STREAM = "tmp_paasta_oom_events"
+
+
+def compose_check_name_for_service_instance(check_name, service, instance):
+    return f"{check_name}.{service}.{instance}"
+
+
+def parse_args(args):
+    parser = argparse.ArgumentParser(
+        description=(
+            "Check the %s stream and report to Sensu if"
+            " there are any OOM events." % OOM_EVENTS_STREAM
+        )
+    )
+    parser.add_argument(
+        "-d",
+        "--soa-dir",
+        dest="soa_dir",
+        default=DEFAULT_SOA_DIR,
+        help="define a different soa config directory",
+    )
+    parser.add_argument(
+        "-r",
+        "--realert-every",
+        dest="realert_every",
+        type=int,
+        default=1,
+        help="Sensu 'realert_every' to use.",
+    )
+    parser.add_argument(
+        "--check-interval",
+        dest="check_interval",
+        type=int,
+        default=1,
+        help="How often this check runs, in minutes.",
+    )
+    parser.add_argument(
+        "--alert-threshold",
+        dest="alert_threshold",
+        type=int,
+        default=1,
+        help="Number of OOM kills required in the check interval to send an alert.",
+    )
+    parser.add_argument(
+        "-s",
+        "--superregion",
+        dest="superregion",
+        required=True,
+        help="The superregion to read OOM events from.",
+    )
+    parser.add_argument(
+        "--dry-run",
+        dest="dry_run",
+        action="store_true",
+        help="Print Sensu alert events instead of sending them",
+    )
+    return parser.parse_args(args)
+
+
+def read_oom_events_from_scribe(cluster, superregion, num_lines=1000):
+    """Read the latest 'num_lines' lines from OOM_EVENTS_STREAM and iterate over them."""
+    # paasta configs incls a map for cluster -> env that is expected by scribe
+    log_reader_config = load_system_paasta_config().get_log_reader()
+    cluster_map = log_reader_config["options"]["cluster_map"]
+    scribe_env = cluster_map[cluster]
+
+    # `scribe_env_to_locations` slightly mutates the scribe env based on whether
+    # or not it is in dev or prod
+    host, port = scribereader.get_tail_host_and_port(
+        **scribe_env_to_locations(scribe_env),
+    )
+    stream = scribereader.get_stream_tailer(
+        stream_name=OOM_EVENTS_STREAM,
+        tailing_host=host,
+        tailing_port=port,
+        lines=num_lines,
+        superregion=superregion,
+    )
+    try:
+        for line in stream:
+            try:
+                j = json.loads(line)
+                if j.get("cluster", "") == cluster:
+                    yield j
+            except json.decoder.JSONDecodeError:
+                pass
+    except StreamTailerSetupError as e:
+        if "No data in stream" in str(e):
+            pass
+        else:
+            raise e
+
+
+def latest_oom_events(cluster, superregion, interval=60):
+    """
+    :returns: {(service, instance): [OOMEvent, OOMEvent,...] }
+              if the number of events > 0
+    """
+    start_timestamp = int(time.time()) - interval
+    res = {}
+    for e in read_oom_events_from_scribe(cluster, superregion):
+        if e["timestamp"] > start_timestamp:
+            key = (e["service"], e["instance"])
+            res.setdefault(key, set()).add(e.get("container_id", ""))
+    return res
+
+
+def compose_sensu_status(
+    instance, oom_events, is_check_enabled, alert_threshold, check_interval
+):
+    """
+    :param instance: InstanceConfig
+    :param oom_events: a list of OOMEvents
+    :param is_check_enabled: boolean to indicate whether the check enabled for the instance
+    """
+    interval_string = f"{check_interval} minute(s)"
+    instance_name = f"{instance.service}.{instance.instance}"
+    if not is_check_enabled:
+        return (Status.OK, f"This check is disabled for {instance_name}.")
+    if not oom_events:
+        return (
+            Status.OK,
+            f"No oom events for {instance_name} in the last {interval_string}.",
+        )
+    elif len(oom_events) >= alert_threshold:
+        return (
+            Status.CRITICAL,
+            f"The Out Of Memory killer killed processes for {instance_name} "
+            f"in the last {interval_string}.",
+        )
+    else:
+        # If the number of OOM kills isn't above the alert threshold,
+        # don't send anything. This will keep an alert open if it's already open,
+        # but won't start a new alert if there wasn't one yet
+        return None
+
+
+def send_sensu_event(instance, oom_events, args):
+    """
+    :param instance: InstanceConfig
+    :param oom_events: a list of OOMEvents
+    """
+    check_name = compose_check_name_for_service_instance(
+        "oom-killer", instance.service, instance.instance
+    )
+    monitoring_overrides = instance.get_monitoring()
+    status = compose_sensu_status(
+        instance=instance,
+        oom_events=oom_events,
+        is_check_enabled=monitoring_overrides.get("check_oom_events", True),
+        alert_threshold=args.alert_threshold,
+        check_interval=args.check_interval,
+    )
+    if not status:
+        return
+
+    memory_limit = instance.get_mem()
+    try:
+        memory_limit_str = f"{int(memory_limit)}MB"
+    except ValueError:
+        memory_limit_str = memory_limit
+
+    monitoring_overrides.update(
+        {
+            "page": False,
+            "alert_after": "0m",
+            "realert_every": args.realert_every,
+            "runbook": "y/check-oom-events",
+            "tip": (
+                "Follow the runbook to investigate and rightsize memory usage "
+                f"(curr: {memory_limit_str})"
+            ),
+        }
+    )
+    return monitoring_tools.send_event(
+        service=instance.service,
+        check_name=check_name,
+        overrides=monitoring_overrides,
+        status=status[0],
+        output=status[1],
+        soa_dir=instance.soa_dir,
+        dry_run=args.dry_run,
+    )
+
+
+def main(sys_argv):
+    args = parse_args(sys_argv[1:])
+    cluster = load_system_paasta_config().get_cluster()
+    victims = latest_oom_events(
+        cluster, args.superregion, interval=(60 * args.check_interval)
+    )
+
+    for (service, instance) in get_services_for_cluster(cluster, soa_dir=args.soa_dir):
+        try:
+            instance_config = get_instance_config(
+                service=service,
+                instance=instance,
+                cluster=cluster,
+                load_deployments=False,
+                soa_dir=args.soa_dir,
+            )
+            oom_events = victims.get((service, instance), [])
+            send_sensu_event(instance_config, oom_events, args)
+        except NotImplementedError:  # When instance_type is not supported by get_instance_config
+            pass
+
+
+if __name__ == "__main__":
+    main(sys.argv)