paasta-tools 1.21.3__py3-none-any.whl

paasta_tools-1.21.3.data/scripts/kubernetes_remove_evicted_pods.py

@@ -0,0 +1,164 @@
+#!python
+# Copyright 2015-2019 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage: ./kubernetes_remove_evicted_pods.py [options]
+
+Removes Evicted pods and notifies service owners
+"""
+import argparse
+import logging
+from collections import defaultdict
+from collections import namedtuple
+from typing import Dict
+from typing import List
+from typing import Mapping
+from typing import Sequence
+
+from kubernetes.client import V1DeleteOptions
+from pysensu_yelp import Status
+
+from paasta_tools.kubernetes_tools import get_all_pods
+from paasta_tools.kubernetes_tools import KubeClient
+from paasta_tools.kubernetes_tools import V1Pod
+from paasta_tools.monitoring_tools import send_event
+from paasta_tools.utils import DEFAULT_SOA_DIR
+
+
+log = logging.getLogger(__name__)
+EvictedPod = namedtuple("EvictedPod", ["podname", "namespace", "eviction_msg"])
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Removes evicted pods and notifies service owners"
+    )
+    parser.add_argument(
+        "-d",
+        "--soa-dir",
+        dest="soa_dir",
+        default=DEFAULT_SOA_DIR,
+        help="define a different soa config directory",
+    )
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", dest="verbose", default=False
+    )
+    parser.add_argument(
+        "-n", "--dry-run", action="store_true", dest="dry_run", default=False
+    )
+    args = parser.parse_args()
+    return args
+
+
+def get_evicted_pods(pods: Sequence[V1Pod]) -> Sequence[V1Pod]:
+    return [
+        pod
+        for pod in pods
+        if pod.status.phase == "Failed" and pod.status.reason == "Evicted"
+    ]
+
+
+def get_pod_service(pod: V1Pod) -> str:
+    if pod.metadata.labels is not None:
+        return pod.metadata.labels.get("paasta.yelp.com/service")
+    else:
+        return None
+
+
+def notify_service_owners(
+    services: Mapping[str, Sequence[EvictedPod]],
+    soa_dir: str,
+    dry_run: bool,
+) -> None:
+    check_overrides = {
+        "page": False,
+        "alert_after": "0m",
+        "realert_every": 1,
+        "tip": "Pods can be Evicted if they go over the allowed quota for a given resource. Check the Eviction message to figure out which resource quota was breached",
+    }
+    for service in services.keys():
+        check_name = f"pod-eviction.{service}"
+        check_output = "The following pods have been evicted and will be removed from the cluster:\n"
+        for pod in services[service]:
+            check_output += f"- {pod.podname}: {pod.eviction_msg}\n"
+        if dry_run:
+            log.info(f"Would have notified owners for service {service}")
+        else:
+            log.info(f"Notifying owners for service {service}")
+            send_event(
+                service,
+                check_name,
+                check_overrides,
+                Status.CRITICAL,
+                check_output,
+                soa_dir,
+            )
+
+
+def remove_pods(
+    client: KubeClient,
+    services: Mapping[str, Sequence[EvictedPod]],
+    dry_run: bool,
+) -> None:
+    delete_options = V1DeleteOptions()
+    for service in services:
+        # Do not remove more than 2 pods per run
+        for pod in services[service][0:2]:
+            if dry_run:
+                log.info(f"Would have removed pod {pod.podname}")
+            else:
+                client.core.delete_namespaced_pod(
+                    pod.podname,
+                    pod.namespace,
+                    body=delete_options,
+                    grace_period_seconds=0,
+                    propagation_policy="Background",
+                )
+                log.info(f"Removing pod {pod.podname}")
+
+
+def evicted_pods_per_service(
+    client: KubeClient,
+) -> Mapping[str, Sequence[EvictedPod]]:
+    all_pods = get_all_pods(kube_client=client, namespace="")
+    evicted_pods = get_evicted_pods(all_pods)
+    log.info(f"Pods in evicted state: {[pod.metadata.name for pod in evicted_pods]}")
+    evicted_pods_aggregated: Dict[str, List[EvictedPod]] = defaultdict(list)
+    for pod in evicted_pods:
+        service = get_pod_service(pod)
+        if service:
+            evicted_pods_aggregated[service].append(
+                EvictedPod(
+                    pod.metadata.name, pod.metadata.namespace, pod.status.message
+                )
+            )
+        else:
+            log.info(f"Could not get service name for pod {pod.metadata.name}")
+    return evicted_pods_aggregated
+
+
+def main() -> None:
+    args = parse_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+    kube_client = KubeClient()
+
+    evicted_pods = evicted_pods_per_service(kube_client)
+    remove_pods(kube_client, evicted_pods, args.dry_run)
+
+
+if __name__ == "__main__":
+    main()

paasta_tools-1.21.3.data/scripts/mass-deploy-tag.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# This script was created to help migrate from branches to tags for deployments.
+# While it is probably useless as-is now, it can hopefully serve as a reference for making
+# future bulk changes to services.
+set -e
+
+WORK_DIR=$(mktemp -d --tmpdir=/nail/tmp)
+pushd ${WORK_DIR}
+
+function cleanup {
+	popd
+	rm -rf "${WORK_DIR}"
+}
+trap cleanup EXIT
+
+services=$(paasta list)
+#services="example_service"
+
+
+for service in ${services} ; do
+	echo "Processing ${service}"
+	jq_output=$(jq -r '.v1 | to_entries | .[] | .key + " " + .value.docker_image' /nail/etc/services/${service}/deployments.json)
+	if [ -z "$jq_output" ] ; then
+		echo "${service} has no deployments. Skipping."
+		continue
+	fi
+	# git_repo=$(paasta info -s ${service} | grep -oP 'Git Repo: \K.*$')
+	git_repo=$(script -qc "paasta info -s ${service}" | sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" | grep 'Git Repo: ' | cut -d' ' -f3)
+	default_git_repo=git@github.yelpcorp.com:services/${service}.git
+	echo git clone ${git_repo-${default_git_repo}} ${service}
+	git clone ${git_repo} ${service}
+	unset git_repo
+	cd ${service}
+	while read -r deploy_group sha; do
+		deploy_group=$(echo ${deploy_group} | sed 's/^.*paasta-//')
+		sha=$(echo ${sha} | sed 's/^.*paasta-//')
+		echo "Mapping ${deploy_group} => ${sha}"
+		# echo paasta mark-for-deployment --git-url ${git_repo} --commit ${sha} --deploy-group ${deploy_group} --service ${service}
+		timestamp='00000000T000000'
+		git tag paasta-${deploy_group}-${timestamp}-deploy ${sha} || true
+        done <<< "$jq_output"
+	git push --tags origin master
+	cd -
+done

paasta_tools-1.21.3.data/scripts/mock_patch_checker.py

@@ -0,0 +1,86 @@
+#!python
+import ast
+import sys
+
+
+class MockChecker(ast.NodeVisitor):
+    def __init__(self):
+        self.errors = 0
+        self.init_module_imports()
+
+    def init_module_imports(self):
+        self.imported_patch = False
+        self.imported_mock = False
+
+    def check_files(self, files):
+        for file in files:
+            self.check_file(file)
+
+    def check_file(self, filename):
+        self.current_filename = filename
+        try:
+            with open(filename, "r") as fd:
+                try:
+                    file_ast = ast.parse(fd.read())
+                except SyntaxError as error:
+                    print("SyntaxError on file %s:%d" % (filename, error.lineno))
+                    return
+        except IOError:
+            print("Error opening filename: %s" % filename)
+            return
+        self.init_module_imports()
+        self.visit(file_ast)
+
+    def _call_uses_patch(self, node):
+        try:
+            return node.func.id == "patch"
+        except AttributeError:
+            return False
+
+    def _call_uses_mock_patch(self, node):
+        try:
+            return node.func.value.id == "mock" and node.func.attr == "patch"
+        except AttributeError:
+            return False
+
+    def visit_Import(self, node):
+        if [name for name in node.names if "mock" == name.name]:
+            self.imported_mock = True
+
+    def visit_ImportFrom(self, node):
+        if node.module == "mock" and (
+            name for name in node.names if "patch" == name.name
+        ):
+            self.imported_patch = True
+
+    def visit_Call(self, node):
+        try:
+            if (self.imported_patch and self._call_uses_patch(node)) or (
+                self.imported_mock and self._call_uses_mock_patch(node)
+            ):
+                if not any(
+                    [keyword for keyword in node.keywords if keyword.arg == "autospec"]
+                ):
+                    print(
+                        "%s:%d: Found a mock without an autospec!"
+                        % (self.current_filename, node.lineno)
+                    )
+                    self.errors += 1
+        except AttributeError:
+            pass
+        self.generic_visit(node)
+
+
+def main(filenames):
+    checker = MockChecker()
+    checker.check_files(filenames)
+    if checker.errors == 0:
+        sys.exit(0)
+    else:
+        print("You probably meant to specify 'autospec=True' in these tests.")
+        print("If you really don't want to, specify 'autospec=None'")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main(sys.argv[1:])

paasta_tools-1.21.3.data/scripts/paasta_cleanup_remote_run_resources.py

@@ -0,0 +1,135 @@
+#!python
+# Copyright 2015-2019 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import logging
+from datetime import datetime
+from datetime import timedelta
+from datetime import timezone
+from typing import Any
+from typing import Callable
+from typing import Sequence
+from typing import Tuple
+
+from paasta_tools.kubernetes.remote_run import get_max_job_duration_limit
+from paasta_tools.kubernetes.remote_run import get_remote_run_jobs
+from paasta_tools.kubernetes.remote_run import get_remote_run_role_bindings
+from paasta_tools.kubernetes.remote_run import get_remote_run_roles
+from paasta_tools.kubernetes.remote_run import get_remote_run_service_accounts
+from paasta_tools.kubernetes_tools import get_all_managed_namespaces
+from paasta_tools.kubernetes_tools import KubeClient
+
+
+ListingFuncType = Callable[[KubeClient, str], Sequence[Any]]
+DeletionFuncType = Callable[[str, str], Any]
+log = logging.getLogger(__name__)
+
+
+def clean_namespace(
+    kube_client: KubeClient,
+    namespace: str,
+    auth_age_limit: datetime,
+    job_age_limit: datetime,
+    dry_run: bool = False,
+):
+    """Clean ephemeral remote-run resource in a namespace
+
+    :param KubeClient kube_client: kubernetes client
+    :param str namepsace: kubernetes namespace
+    :param datetime auth_age_limit: expiration time for authentication resources
+    :param datetime job_age_limit: expiration time for job resources
+    :param bool dry_run: delete resources for real or not
+    """
+    dry_run_msg = " (dry_run)" if dry_run else ""
+    cleanup_actions: Sequence[Tuple[DeletionFuncType, ListingFuncType, datetime]] = (
+        (
+            kube_client.core.delete_namespaced_service_account,
+            get_remote_run_service_accounts,
+            auth_age_limit,
+        ),
+        (
+            kube_client.rbac.delete_namespaced_role,
+            get_remote_run_roles,
+            auth_age_limit,
+        ),
+        (
+            kube_client.rbac.delete_namespaced_role_binding,
+            get_remote_run_role_bindings,
+            auth_age_limit,
+        ),
+        (
+            kube_client.batches.delete_namespaced_job,
+            get_remote_run_jobs,
+            job_age_limit,
+        ),
+    )
+    for delete_func, list_func, age_limit in cleanup_actions:
+        for entity in list_func(kube_client, namespace):
+            if (
+                not entity.metadata.name.startswith("remote-run-")
+                or entity.metadata.creation_timestamp > age_limit
+            ):
+                continue
+            log.info(f"Deleting {entity.metadata.name} in {namespace}{dry_run_msg}")
+            if not dry_run:
+                delete_func(entity.metadata.name, namespace)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Clean ephemeral Kubernetes resources created by remote-run invocations",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "--max-age",
+        type=int,
+        default=600,
+        help="Maximum age, in seconds, resources are allowed to have",
+    )
+    parser.add_argument(
+        "-n",
+        "--dry-run",
+        action="store_true",
+        default=False,
+        help="Do not actually delete resources",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        default=False,
+        help="More verbose logging",
+    )
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
+    kube_client = KubeClient()
+    now = datetime.now(tz=timezone.utc)
+    age_limit = now - timedelta(seconds=args.max_age)
+    job_age_limit = now - timedelta(seconds=get_max_job_duration_limit())
+    for namespace in get_all_managed_namespaces(kube_client):
+        clean_namespace(
+            kube_client,
+            namespace,
+            auth_age_limit=age_limit,
+            job_age_limit=job_age_limit,
+            dry_run=args.dry_run,
+        )
+
+
+if __name__ == "__main__":
+    main()

paasta_tools-1.21.3.data/scripts/paasta_cleanup_stale_nodes.py

@@ -0,0 +1,181 @@
+#!python
+# Copyright 2015-2019 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage: ./paasta_cleanup_stale_nodes.py [options]
+
+Removes terminated Kubernetes nodes from the Kubernetes API.
+
+Command line options:
+
+- -v, --verbose: Verbose output
+- -n, --dry-run: Only report what would have been deleted
+"""
+import argparse
+import logging
+import sys
+from typing import List
+from typing import Sequence
+from typing import Tuple
+
+import boto3
+from boto3_type_annotations.ec2 import Client
+from botocore.exceptions import ClientError
+from kubernetes.client import V1DeleteOptions
+from kubernetes.client import V1Node
+from kubernetes.client.rest import ApiException
+
+from paasta_tools.kubernetes_tools import get_all_nodes
+from paasta_tools.kubernetes_tools import KubeClient
+
+log = logging.getLogger(__name__)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Remove terminated Kubernetes nodes")
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", dest="verbose", default=False
+    )
+    parser.add_argument(
+        "-n", "--dry-run", action="store_true", dest="dry_run", default=False
+    )
+    args = parser.parse_args()
+    return args
+
+
+def nodes_for_cleanup(ec2_client: Client, nodes: Sequence[V1Node]) -> List[V1Node]:
+    not_ready = [
+        node
+        for node in nodes
+        if not is_node_ready(node)
+        and "node-role.kubernetes.io/control-plane" not in node.metadata.labels
+    ]
+    terminated = terminated_nodes(ec2_client, not_ready)
+    return terminated
+
+
+def terminated_nodes(ec2_client: Client, nodes: Sequence[V1Node]) -> List[V1Node]:
+    instance_ids = [node.spec.provider_id.split("/")[-1] for node in nodes]
+    # if there are any instances that don't exist in the query to describe_instance_status
+    # then amazon won't return the results for any, so we have to query the instances
+    # one by one
+    statuses = [
+        does_instance_exist(ec2_client, instance_id) for instance_id in instance_ids
+    ]
+    for node, status in zip(nodes, statuses):
+        log.debug(f"{node.metadata.name} exists: {status}")
+
+    return [node for node, status in zip(nodes, statuses) if not status]
+
+
+def does_instance_exist(ec2_client: Client, instance_id: str):
+    try:
+        instance = ec2_client.describe_instance_status(InstanceIds=[instance_id])
+        if instance["InstanceStatuses"]:
+            status = instance["InstanceStatuses"][0]["InstanceState"]["Name"]
+            # see possible values at https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html#EC2.Client.describe_instance_status
+            # 'pending'|'running'|'shutting-down'|'terminated'|'stopping'|'stopped'
+            # it's unlikely that we'll ever be in this situation with a pending node - the common case is that
+            # the node is either running and been marked as not ready, or it's being shutdown
+            if status not in ("running", "pending"):
+                return False
+            return True
+
+        log.debug(
+            f"no instance status in response for {instance_id}; assuming to have been terminated"
+        )
+        return False
+    except ClientError as e:
+        if e.response["Error"]["Code"] == "InvalidInstanceID.NotFound":
+            log.debug(
+                f"instance {instance_id} not found; assuming to have been terminated"
+            )
+            return False
+        else:
+            log.error(f"error fetching instance status for {instance_id}")
+            raise e
+    return True
+
+
+def terminate_nodes(
+    client: KubeClient, nodes: List[str]
+) -> Tuple[List[str], List[Tuple[str, Exception]]]:
+    success = []
+    errors = []
+    for node in nodes:
+        try:
+            body = V1DeleteOptions()
+            client.core.delete_node(node, body=body, propagation_policy="foreground")
+        except ApiException as e:
+            errors.append((node, e))
+            continue
+        success.append(node)
+    return (success, errors)
+
+
+def is_node_ready(node: V1Node) -> bool:
+    for condition in node.status.conditions:
+        if condition.type == "Ready":
+            return condition.status == "True"
+    log.error(
+        f"no KubeletReady condition found for node {node.metadata.name}. Conditions {node.status.conditions}"
+    )
+    return True
+
+
+def main() -> None:
+    args = parse_args()
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+
+    dry_run = args.dry_run
+
+    kube_client = KubeClient()
+    all_nodes = get_all_nodes(kube_client)
+    log.debug(f"found nodes in cluster {[node.metadata.name for node in all_nodes]}")
+
+    # we depend on iam credentials existing on the host for this to run.
+    # anywhere else, and you'll need to set credentials using environment variables
+    # we also make the assumption that all nodes are in the same region here
+    region = all_nodes[0].metadata.labels["failure-domain.beta.kubernetes.io/region"]
+    ec2_client = boto3.client("ec2", region)
+
+    filtered_nodes = nodes_for_cleanup(ec2_client, all_nodes)
+    if logging.DEBUG >= logging.root.level:
+        log.debug(
+            f"nodes to be deleted: {[node.metadata.name for node in filtered_nodes]}"
+        )
+
+    if not dry_run:
+        success, errors = terminate_nodes(
+            kube_client, [node.metadata.name for node in filtered_nodes]
+        )
+    else:
+        success, errors = [], []
+        log.info("dry run mode detected: not deleting nodes")
+
+    for node_name in success:
+        log.info(f"successfully deleted node {node_name}")
+
+    for node_name, exception in errors:
+        log.error(f"error deleting node: {node_name}: {exception}")
+
+    if errors:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

paasta_tools-1.21.3.data/scripts/paasta_deploy_tron_jobs

@@ -0,0 +1,3 @@
+#!/bin/bash
+set -eo pipefail
+paasta_list_tron_namespaces | xargs -n 1 -r -P 1 paasta_setup_tron_namespace