paasta-tools 1.21.3__py3-none-any.whl

paasta_tools-1.21.3.data/scripts/check_orphans.py

@@ -0,0 +1,306 @@
+#!python
+import argparse
+import asyncio
+import functools
+import json
+import logging
+import os.path
+import re
+import socket
+import sys
+from collections import defaultdict
+from enum import Enum
+from typing import Any
+from typing import DefaultDict
+from typing import Dict
+from typing import List
+from typing import NamedTuple
+from typing import Optional
+from typing import Set
+from typing import Tuple
+
+from kazoo.client import KazooClient
+from kazoo.exceptions import NoNodeError
+
+from paasta_tools import yaml_tools as yaml
+
+logger = logging.getLogger("check_orphans")
+
+PREFIX = "/smartstack/global/"
+CHUNK_SIZE = 50  # How many concurrent xinetd connections
+DEFAULT_ZK_DISCOVERY_PATH = "/nail/etc/zookeeper_discovery/infrastructure/local.yaml"
+DEFAULT_NERVE_XINETD_PORT = 8735
+
+
+class ExitCode(Enum):
+    OK = 0
+    ORPHANS = 1
+    COLLISIONS = 2
+    UNKNOWN = 3
+
+
+def get_zk_hosts(path: str) -> List[str]:
+    with open(path) as f:
+        x = yaml.safe_load(f)
+    return [f"{host}:{port}" for host, port in x]
+
+
+SmartstackData = Dict[str, Dict[str, Any]]
+
+
+def get_zk_data(ignored_services: Set[str]) -> SmartstackData:
+    logger.info(f"using {DEFAULT_ZK_DISCOVERY_PATH} for zookeeper")
+    zk_hosts = get_zk_hosts(DEFAULT_ZK_DISCOVERY_PATH)
+
+    logger.debug(f"connecting to zk hosts {zk_hosts}")
+    zk = KazooClient(hosts=zk_hosts)
+    zk.start()
+
+    logger.debug(f"pulling smartstack data from zookeeper")
+    zk_data = {}
+    services = zk.get_children(PREFIX)
+    for service in services:
+        if service in ignored_services:
+            continue
+        service_instances = zk.get_children(os.path.join(PREFIX, service))
+        instances_data = {}
+        for instance in service_instances:
+            try:
+                instance_node = zk.get(os.path.join(PREFIX, service, instance))
+            except NoNodeError:
+                continue
+            instances_data[instance] = json.loads(instance_node[0])
+            zk_data[service] = instances_data
+
+    return zk_data
+
+
+class InstanceTuple(NamedTuple):
+    # paasta_host may be different from the service's host if running on k8s.
+    # We need the actual PaaSTA host because the k8s pod does not listen for
+    # xinetd connections.
+    paasta_host: str
+    host: str
+    port: int
+    service: str
+
+
+def read_from_zk_data(registrations: SmartstackData) -> Set[InstanceTuple]:
+    return {
+        InstanceTuple(
+            host_to_ip(instance_data["name"], instance_data["host"]),
+            instance_data["host"],
+            instance_data["port"],
+            service,
+        )
+        for service, instance in registrations.items()
+        for instance_data in instance.values()
+    }
+
+
+@functools.lru_cache()
+def host_to_ip(host: str, fallback: str) -> str:
+    """Try to resolve a host to an IP with a fallback.
+
+    Because DNS resolution is relatively slow and can't be easily performed
+    using asyncio, we cheat a little and use a regex for well-formed hostnames
+    to try to guess the IP without doing real resolution.
+
+    A fallback is needed because in some cases the nerve registration does not
+    match an actual hostname (e.g. "prod-db15" or "prod-splunk-master").
+    """
+    for match in (
+        re.match(r"^(\d+)-(\d+)-(\d+)-(\d+)-", host),
+        re.match(r"^ip-(\d+)-(\d+)-(\d+)-(\d+)", host),
+    ):
+        if match:
+            return ".".join(match.groups())
+    else:
+        try:
+            return socket.gethostbyname(host)
+        except socket.gaierror:
+            return fallback
+
+
+async def transfer_one_file(
+    host: str, port: int = DEFAULT_NERVE_XINETD_PORT
+) -> Tuple[str, Optional[str]]:
+    logger.debug(f"getting file from {host}")
+    try:
+        reader, _ = await asyncio.wait_for(
+            asyncio.open_connection(host=host, port=port, limit=2**32), timeout=1.0
+        )
+        resp = await asyncio.wait_for(reader.read(), timeout=1.0)
+    except (asyncio.TimeoutError, ConnectionRefusedError) as ex:
+        # this is not ununusual because we sometimes advertise hosts from
+        # firewalled subnets where we can't make this connection to get
+        # the file. check y/ipam to see what the subnet means
+        logger.debug(f"error getting file from {host}: {ex!r}")
+        return (host, None)
+
+    return (host, resp.decode())
+
+
+async def gather_files(hosts: Set[str]) -> Dict[str, str]:
+    logger.info("gathering files from {} hosts".format(len(hosts)))
+    tasks = [transfer_one_file(host) for host in hosts]
+    responses = {}
+    for idx in range(0, len(tasks), CHUNK_SIZE):
+        resp = await asyncio.gather(
+            *tasks[idx : idx + CHUNK_SIZE], return_exceptions=True
+        )
+        responses.update(dict(resp))
+    return responses
+
+
+def read_one_nerve_file(nerve_config: str) -> Set[InstanceTuple]:
+    nerve_config = json.loads(nerve_config)
+    return {
+        InstanceTuple(
+            # The "instance_id" configured in nerve's config file is the same
+            # as the "name" attribute in a zookeeper registration (i.e. for
+            # PaaSTA hosts, it will be the hostname of the machine running
+            # nerve). To be able to easily compare the tuples using set
+            # operations, we resolve it to an IP in both places.
+            host_to_ip(nerve_config["instance_id"], service["host"]),
+            service["host"],
+            service["port"],
+            service["zk_path"][len(PREFIX) :],
+        )
+        for service in nerve_config["services"].values()
+        if service["zk_path"].startswith(PREFIX)
+    }
+
+
+def read_nerve_files(
+    nerve_configs: Dict[str, Optional[str]]
+) -> Tuple[Set[InstanceTuple], Set[str]]:
+    instance_set: Set[InstanceTuple] = set()
+    not_found_hosts: Set[str] = set()
+    for host, host_config in nerve_configs.items():
+        if host_config is None:
+            not_found_hosts.add(host)
+        else:
+            instance_set |= read_one_nerve_file(host_config)
+    return instance_set, not_found_hosts
+
+
+def get_instance_data(
+    ignored_services: Set[str],
+) -> Tuple[Set[InstanceTuple], Set[InstanceTuple]]:
+    # Dump ZK
+    zk_data = get_zk_data(ignored_services)
+    zk_instance_data = read_from_zk_data(zk_data)
+
+    hosts = {x[0] for x in zk_instance_data}
+
+    # Dump Nerve configs from each host via xinetd
+    results = asyncio.get_event_loop().run_until_complete(gather_files(hosts))
+
+    nerve_instance_data, not_found_hosts = read_nerve_files(results)
+
+    # Filter out anything that we couldn't get a nerve config for
+    zk_instance_data_filtered = {
+        x for x in zk_instance_data if x[0] not in not_found_hosts
+    }
+
+    logger.info("zk_instance_data (unfiltered) len: {}".format(len(zk_instance_data)))
+    logger.info(
+        "zk_instance_data (filtered) len: {}".format(len(zk_instance_data_filtered))
+    )
+    logger.info("nerve_instance_data len: {}".format(len(nerve_instance_data)))
+
+    return zk_instance_data_filtered, nerve_instance_data
+
+
+def check_orphans(
+    zk_instance_data: Set[InstanceTuple],
+    nerve_instance_data: Set[InstanceTuple],
+    check_orphans: bool,
+    check_collisions: bool,
+) -> ExitCode:
+
+    if check_collisions:
+        # collisions
+        instance_by_addr: DefaultDict[Tuple[str, int], Set[str]] = defaultdict(set)
+        for nerve_inst in nerve_instance_data:
+            instance_by_addr[(nerve_inst.host, nerve_inst.port)].add(nerve_inst.service)
+        collisions: List[str] = []
+        for zk_inst in zk_instance_data:
+            nerve_services = instance_by_addr[(zk_inst.host, zk_inst.port)]
+            if len(nerve_services) >= 1 and zk_inst.service not in nerve_services:
+                collisions.append(
+                    f"[{zk_inst.host}:{zk_inst.port}] {zk_inst.service} collides with {nerve_services}"
+                )
+
+        if collisions:
+            logger.warning("Collisions found! Traffic is being misrouted!")
+            print("\n".join(collisions))
+            return ExitCode.COLLISIONS
+        else:
+            logger.info(
+                f"No collisions found out of {len(zk_instance_data)} service registrations seen."
+            )
+    if check_orphans:
+        orphans = zk_instance_data - nerve_instance_data
+
+        # groupby host
+        orphans_by_host: DefaultDict[str, List[Tuple[int, str]]] = defaultdict(list)
+        for orphan in orphans:
+            orphans_by_host[orphan.host].append((orphan.port, orphan.service))
+
+        if orphans:
+            logger.warning("{} orphans found".format(len(orphans)))
+            print(dict(orphans_by_host))
+            return ExitCode.ORPHANS
+        else:
+            logger.info(
+                f"No orphans found out of {len(zk_instance_data)} service registrations seen."
+            )
+
+    return ExitCode.OK
+
+
+def main() -> ExitCode:
+    logging.basicConfig(level=logging.WARNING)
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--ignored-services",
+        # TODO(ckuehl|2020-08-27): Remove this deprecated option alias eventually.
+        "--blacklisted-services-DEPRECATED",
+        default="",
+        type=str,
+        help="Comma separated list of services to ignore",
+    )
+    parser.add_argument(
+        "--no-check-collisions",
+        default=False,
+        action="store_true",
+        help="Skip checking collisions",
+    )
+    parser.add_argument(
+        "--no-check-orphans",
+        default=False,
+        action="store_true",
+        help="Skip checking orphans",
+    )
+    args = parser.parse_args()
+
+    if args.no_check_collisions and args.no_check_orphans:
+        logger.error("Must check at least one of orphans or collisions.")
+        return ExitCode.UNKNOWN
+
+    zk_instance_data, nerve_instance_data = get_instance_data(
+        set(args.ignored_services.split(","))
+    )
+
+    return check_orphans(
+        zk_instance_data,
+        nerve_instance_data,
+        check_orphans=not args.no_check_orphans,
+        check_collisions=not args.no_check_collisions,
+    )
+
+
+if __name__ == "__main__":
+    sys.exit(main().value)

paasta_tools-1.21.3.data/scripts/check_spark_jobs.py

@@ -0,0 +1,234 @@
+#!python
+import argparse
+import datetime
+import logging
+import smtplib
+import sys
+from collections import defaultdict
+from email.message import EmailMessage
+from socket import getfqdn
+
+import pysensu_yelp
+import requests
+
+from paasta_tools import mesos_tools
+from paasta_tools.monitoring_tools import send_event
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import list_services
+
+
+logger = logging.getLogger(__name__)
+email_from_address = f"paasta@{getfqdn()}"
+
+
+JUPYTER_PREFIX = "jupyterhub_"
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Reports long-running Spark frameworks."
+    )
+    parser.add_argument(
+        "--min-hours",
+        type=float,
+        help="Report frameworks that have been registered for more than this duration",
+        default=0,
+    )
+    parser.add_argument(
+        "--no-notify",
+        action="store_true",
+        help="Skip notifying the teams that own each framework",
+    )
+    parser.add_argument(
+        "--email-domain", default=None, help="Email domain for notifying users"
+    )
+    return parser.parse_args()
+
+
+def get_time_running(framework):
+    registered_time = datetime.datetime.fromtimestamp(framework["registered_time"])
+    return datetime.datetime.now() - registered_time
+
+
+def get_spark_properties(framework):
+    webui_url = framework.get("webui_url")
+    if not webui_url:
+        return None
+
+    env_endpoint = f"{webui_url}/api/v1/applications/{framework.id}/environment"
+    try:
+        response = requests.get(env_endpoint, timeout=5)
+    except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
+        logger.warning(f"Unable to connect to {env_endpoint}: {e!r}")
+        return None
+
+    if response.status_code != 200:
+        logger.warning(f"Bad response from {env_endpoint}: {response.status_code}")
+        return None
+
+    try:
+        return response.json()["sparkProperties"]
+    except (ValueError, KeyError):
+        logger.warning(
+            f"Unable to get sparkProperties for {framework.id}: got response {response.text}"
+        )
+        return None
+
+
+def guess_service(properties):
+    if not properties:
+        return None
+    for key, value in properties:
+        if key == "spark.executorEnv.PAASTA_SERVICE":
+            service = value
+            break
+    else:
+        return None
+    if service.startswith(JUPYTER_PREFIX):
+        return service[len(JUPYTER_PREFIX) :]
+    else:
+        return service
+
+
+def get_matching_framework_info(min_hours):
+    frameworks = mesos_tools.get_all_frameworks(active_only=True)
+    matching_info = []
+    min_timedelta = datetime.timedelta(hours=min_hours)
+    for framework in frameworks:
+        if not framework.active:
+            continue
+        if framework.get("principal") != "spark":
+            continue
+        time_running = get_time_running(framework)
+        if time_running >= min_timedelta:
+            info = {
+                "id": framework.id,
+                "name": framework.name,
+                "webui_url": framework.get("webui_url"),
+                "service": guess_service(get_spark_properties(framework)),
+                "user": framework.user,
+                "time_running": str(time_running),
+            }
+            matching_info.append(info)
+
+    return matching_info
+
+
+def format_framework(info):
+    result = [f'{info["name"]} (running for {info["time_running"]})']
+    result.append(f'  user: {info["user"]}')
+    result.append(f'  job UI: {info["webui_url"]}')
+    return "\n".join(result)
+
+
+def format_message_for_service(service, frameworks):
+    output = f"Found the following long-running Spark frameworks associated with service {service}.\n"
+    output += (
+        f"Please check why they are still running and terminate if appropriate.\n\n"
+    )
+    output += "\n".join(format_framework(f) for f in frameworks)
+    return output
+
+
+def get_messages_by_service(frameworks):
+    frameworks_by_service = defaultdict(list)
+    for framework in frameworks:
+        service = framework["service"]
+        frameworks_by_service[service].append(framework)
+
+    return {
+        service: format_message_for_service(service, frameworks)
+        for service, frameworks in frameworks_by_service.items()
+    }
+
+
+def update_check_status(service, output, status):
+    overrides = {
+        "page": False,
+        "alert_after": 0,
+        "tip": "Ask the user to check the job UI and terminate the job if appropriate.",
+        "runbook": "http://y/spark-debug",
+        "ticket": True,
+    }
+    send_event(
+        service=service,
+        check_name=f"long_running_spark_jobs.{service}",
+        overrides=overrides,
+        status=status,
+        output=output,
+        soa_dir=DEFAULT_SOA_DIR,
+    )
+
+
+def email_user(framework_info, email_domain):
+    guessed_user = None
+    if framework_info["user"] != "root":
+        guessed_user = framework_info["user"]
+    elif framework_info["name"].startswith(JUPYTER_PREFIX):
+        try:
+            # the job format is now `<AppName>_<UserName>_<UIPort>_<StartTime>`
+            guessed_user = framework_info["name"].split("_")[-3]
+        except IndexError:
+            pass
+
+    if guessed_user:
+        print(
+            f'Guessed {framework_info["name"]} belongs to {guessed_user}, sending email'
+        )
+    else:
+        print(f"Could not guess user from {framework_info}, skipping user email")
+        return
+
+    msg = EmailMessage()
+    msg["From"] = email_from_address
+    msg["To"] = f"{guessed_user}@{email_domain}"
+    msg["Subject"] = f'Long-running Spark framework {framework_info["name"]}'
+    content = "Please check why it is still running and terminate if appropriate.\n"
+    content += format_framework(framework_info)
+    msg.set_content(content)
+    with smtplib.SMTP("localhost") as s:
+        s.send_message(msg)
+
+
+def report_spark_jobs(min_hours, no_notify, email_domain=None):
+    frameworks = get_matching_framework_info(min_hours=min_hours)
+    messages_by_service = get_messages_by_service(frameworks)
+    valid_services = set(list_services())
+
+    messages_for_unknown_services = []
+    for service, message in messages_by_service.items():
+        if service in valid_services:
+            print(f"{message}\n")
+        else:
+            messages_for_unknown_services.append(message)
+    if messages_for_unknown_services:
+        print("\nINVALID SERVICES")
+        print("----------------")
+        print(
+            "The following frameworks are associated with services that are not configured in PaaSTA.\n"
+        )
+        print("\n\n".join(messages_for_unknown_services))
+
+    if not no_notify:
+        for service in valid_services:
+            if service in messages_by_service:
+                update_check_status(service, message, pysensu_yelp.Status.WARNING)
+            else:
+                update_check_status(
+                    service, "No long running spark jobs", pysensu_yelp.Status.OK
+                )
+        if email_domain:
+            for framework in frameworks:
+                email_user(framework, email_domain)
+
+    return 0 if len(frameworks) == 0 else 1
+
+
+def main():
+    args = parse_args()
+    logging.basicConfig()
+    return report_spark_jobs(args.min_hours, args.no_notify, args.email_domain)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

paasta_tools-1.21.3.data/scripts/cleanup_kubernetes_cr.py

@@ -0,0 +1,138 @@
+#!python
+# Copyright 2015-2018 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage: ./cleanup_kubernetes_cr.py [options]
+
+Command line options:
+
+- -d <SOA_DIR>, --soa-dir <SOA_DIR>: Specify a SOA config dir to read from
+- -v, --verbose: Verbose output
+"""
+import argparse
+import logging
+import sys
+from typing import Sequence
+
+from paasta_tools.kubernetes_tools import CustomResourceDefinition
+from paasta_tools.kubernetes_tools import delete_custom_resource
+from paasta_tools.kubernetes_tools import KubeClient
+from paasta_tools.kubernetes_tools import list_custom_resources
+from paasta_tools.kubernetes_tools import load_custom_resource_definitions
+from paasta_tools.kubernetes_tools import paasta_prefixed
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import load_all_configs
+from paasta_tools.utils import load_system_paasta_config
+
+log = logging.getLogger(__name__)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Cleanup custom_resources.")
+    parser.add_argument(
+        "-d",
+        "--soa-dir",
+        dest="soa_dir",
+        metavar="SOA_DIR",
+        default=DEFAULT_SOA_DIR,
+        help="define a different soa config directory",
+    )
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", dest="verbose", default=False
+    )
+    parser.add_argument(
+        "-c", "--cluster", default=None, help="Cluster to cleanup CRs for"
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main() -> None:
+    args = parse_args()
+    soa_dir = args.soa_dir
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+
+    kube_client = KubeClient()
+
+    system_paasta_config = load_system_paasta_config()
+    cluster = args.cluster or system_paasta_config.get_cluster()
+    custom_resource_definitions = load_custom_resource_definitions(system_paasta_config)
+    cleanup_kube_succeeded = cleanup_all_custom_resources(
+        kube_client=kube_client,
+        soa_dir=soa_dir,
+        cluster=cluster,
+        custom_resource_definitions=custom_resource_definitions,
+    )
+    sys.exit(0 if cleanup_kube_succeeded else 1)
+
+
+def cleanup_all_custom_resources(
+    kube_client: KubeClient,
+    soa_dir: str,
+    cluster: str,
+    custom_resource_definitions: Sequence[CustomResourceDefinition],
+) -> bool:
+    cluster_crds = {
+        crd.spec.names.kind
+        for crd in kube_client.apiextensions.list_custom_resource_definition(
+            label_selector=paasta_prefixed("service")
+        ).items
+    }
+    log.debug(f"CRDs found: {cluster_crds}")
+    results = []
+    for crd in custom_resource_definitions:
+        if crd.kube_kind.singular not in cluster_crds:
+            # TODO: kube_kind.singular seems to correspond to `crd.names.kind`
+            # and not `crd.names.singular`
+            log.warning(f"CRD {crd.kube_kind.singular} " f"not found in {cluster}")
+            continue
+        config_dicts = load_all_configs(
+            cluster=cluster, file_prefix=crd.file_prefix, soa_dir=soa_dir
+        )
+        if not config_dicts:
+            continue
+        crs = list_custom_resources(
+            kube_client=kube_client,
+            kind=crd.kube_kind,
+            version=crd.version,
+            group=crd.group,
+        )
+        for cr in crs:
+            service = config_dicts.get(cr.service)
+            if service is not None:
+                instance = service.get(cr.instance)
+                if instance is not None:
+                    continue
+            result = False
+            try:
+                delete_custom_resource(
+                    kube_client=kube_client,
+                    name=cr.name,
+                    namespace=cr.namespace,
+                    plural=crd.kube_kind.plural,
+                    version=crd.version,
+                    group=crd.group,
+                )
+                result = True
+            except Exception:
+                log.exception("Error while deleting CR {cr.name}")
+            results.append(result)
+    return all(results) if results else True
+
+
+if __name__ == "__main__":
+    main()