paasta-tools 1.21.3__py3-none-any.whl

paasta_tools/kubernetes/bin/kubernetes_remove_evicted_pods.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python
+# Copyright 2015-2019 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage: ./kubernetes_remove_evicted_pods.py [options]
+
+Removes Evicted pods and notifies service owners
+"""
+import argparse
+import logging
+from collections import defaultdict
+from collections import namedtuple
+from typing import Dict
+from typing import List
+from typing import Mapping
+from typing import Sequence
+
+from kubernetes.client import V1DeleteOptions
+from pysensu_yelp import Status
+
+from paasta_tools.kubernetes_tools import get_all_pods
+from paasta_tools.kubernetes_tools import KubeClient
+from paasta_tools.kubernetes_tools import V1Pod
+from paasta_tools.monitoring_tools import send_event
+from paasta_tools.utils import DEFAULT_SOA_DIR
+
+
+log = logging.getLogger(__name__)
+EvictedPod = namedtuple("EvictedPod", ["podname", "namespace", "eviction_msg"])
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Removes evicted pods and notifies service owners"
+    )
+    parser.add_argument(
+        "-d",
+        "--soa-dir",
+        dest="soa_dir",
+        default=DEFAULT_SOA_DIR,
+        help="define a different soa config directory",
+    )
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", dest="verbose", default=False
+    )
+    parser.add_argument(
+        "-n", "--dry-run", action="store_true", dest="dry_run", default=False
+    )
+    args = parser.parse_args()
+    return args
+
+
+def get_evicted_pods(pods: Sequence[V1Pod]) -> Sequence[V1Pod]:
+    return [
+        pod
+        for pod in pods
+        if pod.status.phase == "Failed" and pod.status.reason == "Evicted"
+    ]
+
+
+def get_pod_service(pod: V1Pod) -> str:
+    if pod.metadata.labels is not None:
+        return pod.metadata.labels.get("paasta.yelp.com/service")
+    else:
+        return None
+
+
+def notify_service_owners(
+    services: Mapping[str, Sequence[EvictedPod]],
+    soa_dir: str,
+    dry_run: bool,
+) -> None:
+    check_overrides = {
+        "page": False,
+        "alert_after": "0m",
+        "realert_every": 1,
+        "tip": "Pods can be Evicted if they go over the allowed quota for a given resource. Check the Eviction message to figure out which resource quota was breached",
+    }
+    for service in services.keys():
+        check_name = f"pod-eviction.{service}"
+        check_output = "The following pods have been evicted and will be removed from the cluster:\n"
+        for pod in services[service]:
+            check_output += f"- {pod.podname}: {pod.eviction_msg}\n"
+        if dry_run:
+            log.info(f"Would have notified owners for service {service}")
+        else:
+            log.info(f"Notifying owners for service {service}")
+            send_event(
+                service,
+                check_name,
+                check_overrides,
+                Status.CRITICAL,
+                check_output,
+                soa_dir,
+            )
+
+
+def remove_pods(
+    client: KubeClient,
+    services: Mapping[str, Sequence[EvictedPod]],
+    dry_run: bool,
+) -> None:
+    delete_options = V1DeleteOptions()
+    for service in services:
+        # Do not remove more than 2 pods per run
+        for pod in services[service][0:2]:
+            if dry_run:
+                log.info(f"Would have removed pod {pod.podname}")
+            else:
+                client.core.delete_namespaced_pod(
+                    pod.podname,
+                    pod.namespace,
+                    body=delete_options,
+                    grace_period_seconds=0,
+                    propagation_policy="Background",
+                )
+                log.info(f"Removing pod {pod.podname}")
+
+
+def evicted_pods_per_service(
+    client: KubeClient,
+) -> Mapping[str, Sequence[EvictedPod]]:
+    all_pods = get_all_pods(kube_client=client, namespace="")
+    evicted_pods = get_evicted_pods(all_pods)
+    log.info(f"Pods in evicted state: {[pod.metadata.name for pod in evicted_pods]}")
+    evicted_pods_aggregated: Dict[str, List[EvictedPod]] = defaultdict(list)
+    for pod in evicted_pods:
+        service = get_pod_service(pod)
+        if service:
+            evicted_pods_aggregated[service].append(
+                EvictedPod(
+                    pod.metadata.name, pod.metadata.namespace, pod.status.message
+                )
+            )
+        else:
+            log.info(f"Could not get service name for pod {pod.metadata.name}")
+    return evicted_pods_aggregated
+
+
+def main() -> None:
+    args = parse_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+    kube_client = KubeClient()
+
+    evicted_pods = evicted_pods_per_service(kube_client)
+    remove_pods(kube_client, evicted_pods, args.dry_run)
+
+
+if __name__ == "__main__":
+    main()

paasta_tools/kubernetes/bin/paasta_cleanup_remote_run_resources.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python
+# Copyright 2015-2019 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import logging
+from datetime import datetime
+from datetime import timedelta
+from datetime import timezone
+from typing import Any
+from typing import Callable
+from typing import Sequence
+from typing import Tuple
+
+from paasta_tools.kubernetes.remote_run import get_max_job_duration_limit
+from paasta_tools.kubernetes.remote_run import get_remote_run_jobs
+from paasta_tools.kubernetes.remote_run import get_remote_run_role_bindings
+from paasta_tools.kubernetes.remote_run import get_remote_run_roles
+from paasta_tools.kubernetes.remote_run import get_remote_run_service_accounts
+from paasta_tools.kubernetes_tools import get_all_managed_namespaces
+from paasta_tools.kubernetes_tools import KubeClient
+
+
+ListingFuncType = Callable[[KubeClient, str], Sequence[Any]]
+DeletionFuncType = Callable[[str, str], Any]
+log = logging.getLogger(__name__)
+
+
+def clean_namespace(
+    kube_client: KubeClient,
+    namespace: str,
+    auth_age_limit: datetime,
+    job_age_limit: datetime,
+    dry_run: bool = False,
+):
+    """Clean ephemeral remote-run resource in a namespace
+
+    :param KubeClient kube_client: kubernetes client
+    :param str namepsace: kubernetes namespace
+    :param datetime auth_age_limit: expiration time for authentication resources
+    :param datetime job_age_limit: expiration time for job resources
+    :param bool dry_run: delete resources for real or not
+    """
+    dry_run_msg = " (dry_run)" if dry_run else ""
+    cleanup_actions: Sequence[Tuple[DeletionFuncType, ListingFuncType, datetime]] = (
+        (
+            kube_client.core.delete_namespaced_service_account,
+            get_remote_run_service_accounts,
+            auth_age_limit,
+        ),
+        (
+            kube_client.rbac.delete_namespaced_role,
+            get_remote_run_roles,
+            auth_age_limit,
+        ),
+        (
+            kube_client.rbac.delete_namespaced_role_binding,
+            get_remote_run_role_bindings,
+            auth_age_limit,
+        ),
+        (
+            kube_client.batches.delete_namespaced_job,
+            get_remote_run_jobs,
+            job_age_limit,
+        ),
+    )
+    for delete_func, list_func, age_limit in cleanup_actions:
+        for entity in list_func(kube_client, namespace):
+            if (
+                not entity.metadata.name.startswith("remote-run-")
+                or entity.metadata.creation_timestamp > age_limit
+            ):
+                continue
+            log.info(f"Deleting {entity.metadata.name} in {namespace}{dry_run_msg}")
+            if not dry_run:
+                delete_func(entity.metadata.name, namespace)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Clean ephemeral Kubernetes resources created by remote-run invocations",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "--max-age",
+        type=int,
+        default=600,
+        help="Maximum age, in seconds, resources are allowed to have",
+    )
+    parser.add_argument(
+        "-n",
+        "--dry-run",
+        action="store_true",
+        default=False,
+        help="Do not actually delete resources",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        default=False,
+        help="More verbose logging",
+    )
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
+    kube_client = KubeClient()
+    now = datetime.now(tz=timezone.utc)
+    age_limit = now - timedelta(seconds=args.max_age)
+    job_age_limit = now - timedelta(seconds=get_max_job_duration_limit())
+    for namespace in get_all_managed_namespaces(kube_client):
+        clean_namespace(
+            kube_client,
+            namespace,
+            auth_age_limit=age_limit,
+            job_age_limit=job_age_limit,
+            dry_run=args.dry_run,
+        )
+
+
+if __name__ == "__main__":
+    main()

paasta_tools/kubernetes/bin/paasta_cleanup_stale_nodes.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python
+# Copyright 2015-2019 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage: ./paasta_cleanup_stale_nodes.py [options]
+
+Removes terminated Kubernetes nodes from the Kubernetes API.
+
+Command line options:
+
+- -v, --verbose: Verbose output
+- -n, --dry-run: Only report what would have been deleted
+"""
+import argparse
+import logging
+import sys
+from typing import List
+from typing import Sequence
+from typing import Tuple
+
+import boto3
+from boto3_type_annotations.ec2 import Client
+from botocore.exceptions import ClientError
+from kubernetes.client import V1DeleteOptions
+from kubernetes.client import V1Node
+from kubernetes.client.rest import ApiException
+
+from paasta_tools.kubernetes_tools import get_all_nodes
+from paasta_tools.kubernetes_tools import KubeClient
+
+log = logging.getLogger(__name__)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Remove terminated Kubernetes nodes")
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", dest="verbose", default=False
+    )
+    parser.add_argument(
+        "-n", "--dry-run", action="store_true", dest="dry_run", default=False
+    )
+    args = parser.parse_args()
+    return args
+
+
+def nodes_for_cleanup(ec2_client: Client, nodes: Sequence[V1Node]) -> List[V1Node]:
+    not_ready = [
+        node
+        for node in nodes
+        if not is_node_ready(node)
+        and "node-role.kubernetes.io/control-plane" not in node.metadata.labels
+    ]
+    terminated = terminated_nodes(ec2_client, not_ready)
+    return terminated
+
+
+def terminated_nodes(ec2_client: Client, nodes: Sequence[V1Node]) -> List[V1Node]:
+    instance_ids = [node.spec.provider_id.split("/")[-1] for node in nodes]
+    # if there are any instances that don't exist in the query to describe_instance_status
+    # then amazon won't return the results for any, so we have to query the instances
+    # one by one
+    statuses = [
+        does_instance_exist(ec2_client, instance_id) for instance_id in instance_ids
+    ]
+    for node, status in zip(nodes, statuses):
+        log.debug(f"{node.metadata.name} exists: {status}")
+
+    return [node for node, status in zip(nodes, statuses) if not status]
+
+
+def does_instance_exist(ec2_client: Client, instance_id: str):
+    try:
+        instance = ec2_client.describe_instance_status(InstanceIds=[instance_id])
+        if instance["InstanceStatuses"]:
+            status = instance["InstanceStatuses"][0]["InstanceState"]["Name"]
+            # see possible values at https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html#EC2.Client.describe_instance_status
+            # 'pending'|'running'|'shutting-down'|'terminated'|'stopping'|'stopped'
+            # it's unlikely that we'll ever be in this situation with a pending node - the common case is that
+            # the node is either running and been marked as not ready, or it's being shutdown
+            if status not in ("running", "pending"):
+                return False
+            return True
+
+        log.debug(
+            f"no instance status in response for {instance_id}; assuming to have been terminated"
+        )
+        return False
+    except ClientError as e:
+        if e.response["Error"]["Code"] == "InvalidInstanceID.NotFound":
+            log.debug(
+                f"instance {instance_id} not found; assuming to have been terminated"
+            )
+            return False
+        else:
+            log.error(f"error fetching instance status for {instance_id}")
+            raise e
+    return True
+
+
+def terminate_nodes(
+    client: KubeClient, nodes: List[str]
+) -> Tuple[List[str], List[Tuple[str, Exception]]]:
+    success = []
+    errors = []
+    for node in nodes:
+        try:
+            body = V1DeleteOptions()
+            client.core.delete_node(node, body=body, propagation_policy="foreground")
+        except ApiException as e:
+            errors.append((node, e))
+            continue
+        success.append(node)
+    return (success, errors)
+
+
+def is_node_ready(node: V1Node) -> bool:
+    for condition in node.status.conditions:
+        if condition.type == "Ready":
+            return condition.status == "True"
+    log.error(
+        f"no KubeletReady condition found for node {node.metadata.name}. Conditions {node.status.conditions}"
+    )
+    return True
+
+
+def main() -> None:
+    args = parse_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+
+    dry_run = args.dry_run
+
+    kube_client = KubeClient()
+    all_nodes = get_all_nodes(kube_client)
+    log.debug(f"found nodes in cluster {[node.metadata.name for node in all_nodes]}")
+
+    # we depend on iam credentials existing on the host for this to run.
+    # anywhere else, and you'll need to set credentials using environment variables
+    # we also make the assumption that all nodes are in the same region here
+    region = all_nodes[0].metadata.labels["failure-domain.beta.kubernetes.io/region"]
+    ec2_client = boto3.client("ec2", region)
+
+    filtered_nodes = nodes_for_cleanup(ec2_client, all_nodes)
+    if logging.DEBUG >= logging.root.level:
+        log.debug(
+            f"nodes to be deleted: {[node.metadata.name for node in filtered_nodes]}"
+        )
+
+    if not dry_run:
+        success, errors = terminate_nodes(
+            kube_client, [node.metadata.name for node in filtered_nodes]
+        )
+    else:
+        success, errors = [], []
+        log.info("dry run mode detected: not deleting nodes")
+
+    for node_name in success:
+        log.info(f"successfully deleted node {node_name}")
+
+    for node_name, exception in errors:
+        log.error(f"error deleting node: {node_name}: {exception}")
+
+    if errors:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()