paasta-tools 1.21.3__py3-none-any.whl

paasta_tools/check_services_replication_tools.py

@@ -0,0 +1,324 @@
+# Copyright 2015-2019 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import logging
+import sys
+from multiprocessing import Pool
+from os import cpu_count
+from typing import Any
+from typing import Callable
+from typing import cast
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Sequence
+from typing import Set
+from typing import Tuple
+from typing import Type
+
+from mypy_extensions import Arg
+from mypy_extensions import NamedArg
+
+from paasta_tools.kubernetes_tools import get_all_managed_namespaces
+from paasta_tools.kubernetes_tools import get_all_nodes
+from paasta_tools.kubernetes_tools import get_all_pods
+from paasta_tools.kubernetes_tools import group_pods_by_service_instance
+from paasta_tools.kubernetes_tools import KubeClient
+from paasta_tools.kubernetes_tools import V1Node
+from paasta_tools.kubernetes_tools import V1Pod
+from paasta_tools.metrics import metrics_lib
+from paasta_tools.monitoring_tools import ReplicationChecker
+from paasta_tools.paasta_service_config_loader import PaastaServiceConfigLoader
+from paasta_tools.smartstack_tools import KubeSmartstackEnvoyReplicationChecker
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import InstanceConfig_T
+from paasta_tools.utils import list_services
+from paasta_tools.utils import load_system_paasta_config
+from paasta_tools.utils import SPACER
+
+try:
+    import yelp_meteorite
+except ImportError:
+    yelp_meteorite = None
+
+log = logging.getLogger(__name__)
+
+CheckServiceReplication = Callable[
+    [
+        Arg(InstanceConfig_T, "instance_config"),
+        Arg(Dict[str, Dict[str, List[V1Pod]]], "pods_by_service_instance"),
+        Arg(Any, "replication_checker"),
+        NamedArg(bool, "dry_run"),
+    ],
+    Optional[bool],
+]
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-d",
+        "--soa-dir",
+        dest="soa_dir",
+        metavar="SOA_DIR",
+        default=DEFAULT_SOA_DIR,
+        help="define a different soa config directory",
+    )
+    parser.add_argument(
+        "--crit",
+        dest="under_replicated_crit_pct",
+        type=float,
+        default=10,
+        help="The percentage of under replicated service instances past which "
+        "the script will return a critical status",
+    )
+    parser.add_argument(
+        "--min-count-critical",
+        dest="min_count_critical",
+        type=int,
+        default=5,
+        help="The script will not return a critical status if the number of "
+        "under replicated service instances is below this number, even if the "
+        "percentage is above the critical percentage.",
+    )
+    parser.add_argument(
+        "service_instance_list",
+        nargs="*",
+        help="The list of service instances to check",
+        metavar="SERVICE%sINSTANCE" % SPACER,
+    )
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", dest="verbose", default=False
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        dest="dry_run",
+        help="Print Sensu alert events and metrics instead of sending them",
+    )
+    parser.add_argument(
+        "--eks",
+        help="This flag checks k8 services running on EKS",
+        dest="eks",
+        action="store_true",
+        default=False,
+    )
+    options = parser.parse_args()
+
+    return options
+
+
+def check_services_replication(
+    soa_dir: str,
+    cluster: str,
+    service_instances: Sequence[str],
+    instance_type_class: Type[InstanceConfig_T],
+    check_service_replication: CheckServiceReplication,
+    replication_checker: ReplicationChecker,
+    pods_by_service_instance: Dict[str, Dict[str, List[V1Pod]]],
+    dry_run: bool = False,
+) -> Tuple[int, int]:
+    service_instances_set = set(service_instances)
+    replication_statuses: List[bool] = []
+
+    for service in list_services(soa_dir=soa_dir):
+        service_config = PaastaServiceConfigLoader(service=service, soa_dir=soa_dir)
+        for instance_config in service_config.instance_configs(
+            cluster=cluster, instance_type_class=instance_type_class
+        ):
+            if (
+                service_instances_set
+                and f"{service}{SPACER}{instance_config.instance}"
+                not in service_instances_set
+            ):
+                continue
+            if instance_config.get_docker_image():
+                is_well_replicated = check_service_replication(
+                    instance_config=instance_config,
+                    pods_by_service_instance=pods_by_service_instance,
+                    replication_checker=replication_checker,
+                    dry_run=dry_run,
+                )
+                if is_well_replicated is not None:
+                    replication_statuses.append(is_well_replicated)
+
+            else:
+                log.debug(
+                    "%s is not deployed. Skipping replication monitoring."
+                    % instance_config.job_id
+                )
+
+    num_under_replicated = len(
+        [status for status in replication_statuses if status is False]
+    )
+    return num_under_replicated, len(replication_statuses)
+
+
+def emit_cluster_replication_metrics(
+    pct_under_replicated: float,
+    cluster: str,
+    scheduler: str,
+    dry_run: bool = False,
+) -> None:
+    metric_name = "paasta.pct_services_under_replicated"
+    if dry_run:
+        print(f"Would've sent value {pct_under_replicated} for metric '{metric_name}'")
+    else:
+        meteorite_dims = {"paasta_cluster": cluster, "scheduler": scheduler}
+        gauge = yelp_meteorite.create_gauge(metric_name, meteorite_dims)
+        gauge.set(pct_under_replicated)
+
+
+def main(
+    instance_type_class: Type[InstanceConfig_T],
+    check_service_replication: CheckServiceReplication,
+    namespace: str = None,
+) -> None:
+    args = parse_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.WARNING)
+
+    system_paasta_config = load_system_paasta_config()
+    cluster = system_paasta_config.get_cluster()
+    replication_checker: ReplicationChecker
+
+    timer = metrics_lib.system_timer(dimensions=dict(eks=args.eks, cluster=cluster))
+
+    timer.start()
+
+    if namespace:
+        pods, nodes = get_kubernetes_pods_and_nodes(namespace=namespace)
+        replication_checker = KubeSmartstackEnvoyReplicationChecker(
+            nodes=nodes,
+            system_paasta_config=system_paasta_config,
+        )
+    else:
+        pods, nodes = get_kubernetes_pods_and_nodes()
+        replication_checker = KubeSmartstackEnvoyReplicationChecker(
+            nodes=nodes,
+            system_paasta_config=system_paasta_config,
+        )
+
+    pods_by_service_instance = group_pods_by_service_instance(pods)
+
+    count_under_replicated, total = check_services_replication(
+        soa_dir=args.soa_dir,
+        cluster=cluster,
+        service_instances=args.service_instance_list,
+        instance_type_class=instance_type_class,
+        check_service_replication=check_service_replication,
+        replication_checker=replication_checker,
+        pods_by_service_instance=pods_by_service_instance,
+        dry_run=args.dry_run,
+    )
+    pct_under_replicated = 0 if total == 0 else 100 * count_under_replicated / total
+    if yelp_meteorite is not None:
+        emit_cluster_replication_metrics(
+            pct_under_replicated,
+            cluster,
+            scheduler="kubernetes",
+            dry_run=args.dry_run,
+        )
+
+    exit_code = 0
+    if (
+        pct_under_replicated >= args.under_replicated_crit_pct
+        and count_under_replicated >= args.min_count_critical
+    ):
+        log.critical(
+            f"{pct_under_replicated}% of instances ({count_under_replicated}/{total}) "
+            f"are under replicated (past {args.under_replicated_crit_pct} is critical)!"
+        )
+        exit_code = 2
+
+    timer.stop(tmp_dimensions={"result": exit_code})
+    logging.info(
+        f"Stopping timer for {cluster} (eks={args.eks}) with result {exit_code}: {timer()}ms elapsed"
+    )
+    sys.exit(exit_code)
+
+
+# XXX: is there a base class for the k8s clientlib models that we could use to type `obj`?
+def set_local_vars_configuration_to_none(obj: Any, visited: Set[int] = None) -> None:
+    """
+    Recursive function to ensure that k8s clientlib objects are pickleable.
+
+    Without this, k8s clientlib objects can't be used by multiprocessing functions
+    as those pickle data to shuttle between processes.
+    """
+    if visited is None:
+        visited = set()
+
+    # Avoid infinite recursion for objects that have already been visited
+    obj_id = id(obj)
+    if obj_id in visited:
+        return
+    visited.add(obj_id)
+
+    # if the object has the attribute, set it to None to essentially delete it
+    if hasattr(obj, "local_vars_configuration"):
+        setattr(obj, "local_vars_configuration", None)
+
+    # recursively check attributes of the object
+    if hasattr(obj, "__dict__"):
+        for attr_name, attr_value in obj.__dict__.items():
+            set_local_vars_configuration_to_none(attr_value, visited)
+
+    # if the object is iterable/a collection, iterate over its elements
+    elif isinstance(obj, (list, tuple, set)):
+        for item in obj:
+            set_local_vars_configuration_to_none(item, visited)
+    elif isinstance(obj, dict):
+        for value in obj.values():
+            set_local_vars_configuration_to_none(value, visited)
+
+
+def __fetch_pods(namespace: str) -> List[V1Pod]:
+    kube_client = KubeClient()
+    pods = get_all_pods(kube_client, namespace)
+    for pod in pods:
+        # this is pretty silly, but V1Pod cannot be pickled otherwise since the local_vars_configuration member
+        # is not picklable - and pretty much every k8s model has this member ;_;
+        set_local_vars_configuration_to_none(pod)
+    return pods
+
+
+def __get_all_pods_parallel(from_namespaces: Set[str]) -> List[V1Pod]:
+    all_pods: List[V1Pod] = []
+    with Pool() as pool:
+        for pod_list in pool.imap_unordered(
+            __fetch_pods,
+            from_namespaces,
+            chunksize=len(from_namespaces) // cast(int, cpu_count()),
+        ):
+            all_pods.extend(pod_list)
+    return all_pods
+
+
+def get_kubernetes_pods_and_nodes(
+    namespace: Optional[str] = None,
+) -> Tuple[List[V1Pod], List[V1Node]]:
+    kube_client = KubeClient()
+
+    if namespace:
+        all_pods = get_all_pods(kube_client=kube_client, namespace=namespace)
+    else:
+        all_managed_namespaces = set(get_all_managed_namespaces(kube_client))
+        all_pods = __get_all_pods_parallel(all_managed_namespaces)
+
+    all_nodes = get_all_nodes(kube_client)
+
+    return all_pods, all_nodes

paasta_tools/check_spark_jobs.py

@@ -0,0 +1,234 @@
+#!/usr/bin/env python
+import argparse
+import datetime
+import logging
+import smtplib
+import sys
+from collections import defaultdict
+from email.message import EmailMessage
+from socket import getfqdn
+
+import pysensu_yelp
+import requests
+
+from paasta_tools import mesos_tools
+from paasta_tools.monitoring_tools import send_event
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import list_services
+
+
+logger = logging.getLogger(__name__)
+email_from_address = f"paasta@{getfqdn()}"
+
+
+JUPYTER_PREFIX = "jupyterhub_"
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Reports long-running Spark frameworks."
+    )
+    parser.add_argument(
+        "--min-hours",
+        type=float,
+        help="Report frameworks that have been registered for more than this duration",
+        default=0,
+    )
+    parser.add_argument(
+        "--no-notify",
+        action="store_true",
+        help="Skip notifying the teams that own each framework",
+    )
+    parser.add_argument(
+        "--email-domain", default=None, help="Email domain for notifying users"
+    )
+    return parser.parse_args()
+
+
+def get_time_running(framework):
+    registered_time = datetime.datetime.fromtimestamp(framework["registered_time"])
+    return datetime.datetime.now() - registered_time
+
+
+def get_spark_properties(framework):
+    webui_url = framework.get("webui_url")
+    if not webui_url:
+        return None
+
+    env_endpoint = f"{webui_url}/api/v1/applications/{framework.id}/environment"
+    try:
+        response = requests.get(env_endpoint, timeout=5)
+    except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
+        logger.warning(f"Unable to connect to {env_endpoint}: {e!r}")
+        return None
+
+    if response.status_code != 200:
+        logger.warning(f"Bad response from {env_endpoint}: {response.status_code}")
+        return None
+
+    try:
+        return response.json()["sparkProperties"]
+    except (ValueError, KeyError):
+        logger.warning(
+            f"Unable to get sparkProperties for {framework.id}: got response {response.text}"
+        )
+        return None
+
+
+def guess_service(properties):
+    if not properties:
+        return None
+    for key, value in properties:
+        if key == "spark.executorEnv.PAASTA_SERVICE":
+            service = value
+            break
+    else:
+        return None
+    if service.startswith(JUPYTER_PREFIX):
+        return service[len(JUPYTER_PREFIX) :]
+    else:
+        return service
+
+
+def get_matching_framework_info(min_hours):
+    frameworks = mesos_tools.get_all_frameworks(active_only=True)
+    matching_info = []
+    min_timedelta = datetime.timedelta(hours=min_hours)
+    for framework in frameworks:
+        if not framework.active:
+            continue
+        if framework.get("principal") != "spark":
+            continue
+        time_running = get_time_running(framework)
+        if time_running >= min_timedelta:
+            info = {
+                "id": framework.id,
+                "name": framework.name,
+                "webui_url": framework.get("webui_url"),
+                "service": guess_service(get_spark_properties(framework)),
+                "user": framework.user,
+                "time_running": str(time_running),
+            }
+            matching_info.append(info)
+
+    return matching_info
+
+
+def format_framework(info):
+    result = [f'{info["name"]} (running for {info["time_running"]})']
+    result.append(f'  user: {info["user"]}')
+    result.append(f'  job UI: {info["webui_url"]}')
+    return "\n".join(result)
+
+
+def format_message_for_service(service, frameworks):
+    output = f"Found the following long-running Spark frameworks associated with service {service}.\n"
+    output += (
+        f"Please check why they are still running and terminate if appropriate.\n\n"
+    )
+    output += "\n".join(format_framework(f) for f in frameworks)
+    return output
+
+
+def get_messages_by_service(frameworks):
+    frameworks_by_service = defaultdict(list)
+    for framework in frameworks:
+        service = framework["service"]
+        frameworks_by_service[service].append(framework)
+
+    return {
+        service: format_message_for_service(service, frameworks)
+        for service, frameworks in frameworks_by_service.items()
+    }
+
+
+def update_check_status(service, output, status):
+    overrides = {
+        "page": False,
+        "alert_after": 0,
+        "tip": "Ask the user to check the job UI and terminate the job if appropriate.",
+        "runbook": "http://y/spark-debug",
+        "ticket": True,
+    }
+    send_event(
+        service=service,
+        check_name=f"long_running_spark_jobs.{service}",
+        overrides=overrides,
+        status=status,
+        output=output,
+        soa_dir=DEFAULT_SOA_DIR,
+    )
+
+
+def email_user(framework_info, email_domain):
+    guessed_user = None
+    if framework_info["user"] != "root":
+        guessed_user = framework_info["user"]
+    elif framework_info["name"].startswith(JUPYTER_PREFIX):
+        try:
+            # the job format is now `<AppName>_<UserName>_<UIPort>_<StartTime>`
+            guessed_user = framework_info["name"].split("_")[-3]
+        except IndexError:
+            pass
+
+    if guessed_user:
+        print(
+            f'Guessed {framework_info["name"]} belongs to {guessed_user}, sending email'
+        )
+    else:
+        print(f"Could not guess user from {framework_info}, skipping user email")
+        return
+
+    msg = EmailMessage()
+    msg["From"] = email_from_address
+    msg["To"] = f"{guessed_user}@{email_domain}"
+    msg["Subject"] = f'Long-running Spark framework {framework_info["name"]}'
+    content = "Please check why it is still running and terminate if appropriate.\n"
+    content += format_framework(framework_info)
+    msg.set_content(content)
+    with smtplib.SMTP("localhost") as s:
+        s.send_message(msg)
+
+
+def report_spark_jobs(min_hours, no_notify, email_domain=None):
+    frameworks = get_matching_framework_info(min_hours=min_hours)
+    messages_by_service = get_messages_by_service(frameworks)
+    valid_services = set(list_services())
+
+    messages_for_unknown_services = []
+    for service, message in messages_by_service.items():
+        if service in valid_services:
+            print(f"{message}\n")
+        else:
+            messages_for_unknown_services.append(message)
+    if messages_for_unknown_services:
+        print("\nINVALID SERVICES")
+        print("----------------")
+        print(
+            "The following frameworks are associated with services that are not configured in PaaSTA.\n"
+        )
+        print("\n\n".join(messages_for_unknown_services))
+
+    if not no_notify:
+        for service in valid_services:
+            if service in messages_by_service:
+                update_check_status(service, message, pysensu_yelp.Status.WARNING)
+            else:
+                update_check_status(
+                    service, "No long running spark jobs", pysensu_yelp.Status.OK
+                )
+        if email_domain:
+            for framework in frameworks:
+                email_user(framework, email_domain)
+
+    return 0 if len(frameworks) == 0 else 1
+
+
+def main():
+    args = parse_args()
+    logging.basicConfig()
+    return report_spark_jobs(args.min_hours, args.no_notify, args.email_domain)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

paasta_tools/cleanup_kubernetes_cr.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python
+# Copyright 2015-2018 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage: ./cleanup_kubernetes_cr.py [options]
+
+Command line options:
+
+- -d <SOA_DIR>, --soa-dir <SOA_DIR>: Specify a SOA config dir to read from
+- -v, --verbose: Verbose output
+"""
+import argparse
+import logging
+import sys
+from typing import Sequence
+
+from paasta_tools.kubernetes_tools import CustomResourceDefinition
+from paasta_tools.kubernetes_tools import delete_custom_resource
+from paasta_tools.kubernetes_tools import KubeClient
+from paasta_tools.kubernetes_tools import list_custom_resources
+from paasta_tools.kubernetes_tools import load_custom_resource_definitions
+from paasta_tools.kubernetes_tools import paasta_prefixed
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import load_all_configs
+from paasta_tools.utils import load_system_paasta_config
+
+log = logging.getLogger(__name__)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Cleanup custom_resources.")
+    parser.add_argument(
+        "-d",
+        "--soa-dir",
+        dest="soa_dir",
+        metavar="SOA_DIR",
+        default=DEFAULT_SOA_DIR,
+        help="define a different soa config directory",
+    )
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", dest="verbose", default=False
+    )
+    parser.add_argument(
+        "-c", "--cluster", default=None, help="Cluster to cleanup CRs for"
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main() -> None:
+    args = parse_args()
+    soa_dir = args.soa_dir
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+
+    kube_client = KubeClient()
+
+    system_paasta_config = load_system_paasta_config()
+    cluster = args.cluster or system_paasta_config.get_cluster()
+    custom_resource_definitions = load_custom_resource_definitions(system_paasta_config)
+    cleanup_kube_succeeded = cleanup_all_custom_resources(
+        kube_client=kube_client,
+        soa_dir=soa_dir,
+        cluster=cluster,
+        custom_resource_definitions=custom_resource_definitions,
+    )
+    sys.exit(0 if cleanup_kube_succeeded else 1)
+
+
+def cleanup_all_custom_resources(
+    kube_client: KubeClient,
+    soa_dir: str,
+    cluster: str,
+    custom_resource_definitions: Sequence[CustomResourceDefinition],
+) -> bool:
+    cluster_crds = {
+        crd.spec.names.kind
+        for crd in kube_client.apiextensions.list_custom_resource_definition(
+            label_selector=paasta_prefixed("service")
+        ).items
+    }
+    log.debug(f"CRDs found: {cluster_crds}")
+    results = []
+    for crd in custom_resource_definitions:
+        if crd.kube_kind.singular not in cluster_crds:
+            # TODO: kube_kind.singular seems to correspond to `crd.names.kind`
+            # and not `crd.names.singular`
+            log.warning(f"CRD {crd.kube_kind.singular} " f"not found in {cluster}")
+            continue
+        config_dicts = load_all_configs(
+            cluster=cluster, file_prefix=crd.file_prefix, soa_dir=soa_dir
+        )
+        if not config_dicts:
+            continue
+        crs = list_custom_resources(
+            kube_client=kube_client,
+            kind=crd.kube_kind,
+            version=crd.version,
+            group=crd.group,
+        )
+        for cr in crs:
+            service = config_dicts.get(cr.service)
+            if service is not None:
+                instance = service.get(cr.instance)
+                if instance is not None:
+                    continue
+            result = False
+            try:
+                delete_custom_resource(
+                    kube_client=kube_client,
+                    name=cr.name,
+                    namespace=cr.namespace,
+                    plural=crd.kube_kind.plural,
+                    version=crd.version,
+                    group=crd.group,
+                )
+                result = True
+            except Exception:
+                log.exception("Error while deleting CR {cr.name}")
+            results.append(result)
+    return all(results) if results else True
+
+
+if __name__ == "__main__":
+    main()