paasta-tools 1.21.3__py3-none-any.whl

paasta_tools/cleanup_kubernetes_crd.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python
+# Copyright 2015-2019 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage: ./cleanup_kubernetes_crd.py [options]
+
+Command line options:
+
+- -d <SOA_DIR>, --soa-dir <SOA_DIR>: Specify a SOA config dir to read from
+- -c <cluster>, --cluster <cluster>: Specify a kubernetes cluster name
+- -v, --verbose: Verbose output
+- -n, --dry-run: Only report what would have been deleted
+"""
+import argparse
+import logging
+import sys
+
+import service_configuration_lib
+from kubernetes.client import V1DeleteOptions
+from kubernetes.client.rest import ApiException
+
+from paasta_tools.kubernetes_tools import KubeClient
+from paasta_tools.kubernetes_tools import paasta_prefixed
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import load_system_paasta_config
+
+log = logging.getLogger(__name__)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Removes stale kubernetes CRDs.")
+    parser.add_argument(
+        "-c",
+        "--cluster",
+        dest="cluster",
+        metavar="CLUSTER",
+        default=None,
+        help="Kubernetes cluster name",
+    )
+    parser.add_argument(
+        "-d",
+        "--soa-dir",
+        dest="soa_dir",
+        metavar="SOA_DIR",
+        default=DEFAULT_SOA_DIR,
+        help="define a different soa config directory",
+    )
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", dest="verbose", default=False
+    )
+    parser.add_argument(
+        "-n", "--dry-run", action="store_true", dest="dry_run", default=False
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main() -> None:
+    args = parse_args()
+    soa_dir = args.soa_dir
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.WARNING)
+
+    if args.cluster:
+        cluster = args.cluster
+    else:
+        system_paasta_config = load_system_paasta_config()
+        cluster = system_paasta_config.get_cluster()
+
+    kube_client = KubeClient()
+
+    success = cleanup_kube_crd(
+        kube_client=kube_client, cluster=cluster, soa_dir=soa_dir, dry_run=args.dry_run
+    )
+    sys.exit(0 if success else 1)
+
+
+def cleanup_kube_crd(
+    kube_client: KubeClient,
+    cluster: str,
+    soa_dir: str = DEFAULT_SOA_DIR,
+    dry_run: bool = False,
+) -> bool:
+    service_attr = paasta_prefixed("service")
+    existing_crds = kube_client.apiextensions.list_custom_resource_definition(
+        label_selector=service_attr
+    )
+
+    success = True
+    for crd in existing_crds.items:
+        service = crd.metadata.labels[service_attr]
+        if not service:
+            log.error(f"CRD {crd.metadata.name} has empty {service_attr} label")
+            continue
+
+        protected_attr = paasta_prefixed("protected")
+        if crd.metadata.labels.get(protected_attr) is not None:
+            log.info(
+                f"CRD {crd.metadata.name} has {protected_attr} label set - skipping."
+            )
+            continue
+
+        crd_config = service_configuration_lib.read_extra_service_information(
+            service, f"crd-{cluster}", soa_dir=soa_dir
+        )
+        if crd_config:
+            log.debug(f"CRD {crd.metadata.name} declaration found in {service}")
+            continue
+
+        log.info(f"CRD {crd.metadata.name} not found in {service} service")
+        if dry_run:
+            log.info("not deleting in dry-run mode")
+            continue
+
+        try:
+            kube_client.apiextensions.delete_custom_resource_definition(
+                name=crd.metadata.name, body=V1DeleteOptions()
+            )
+            log.info(f"deleted {crd.metadata.name} for {cluster}:{service}")
+        except ApiException as exc:
+            log.error(
+                f"error deploying crd for {cluster}:{service}, "
+                f"status: {exc.status}, reason: {exc.reason}"
+            )
+            log.debug(exc.body)
+            success = False
+
+    return success
+
+
+if __name__ == "__main__":
+    main()

paasta_tools/cleanup_kubernetes_jobs.py

@@ -0,0 +1,344 @@
+#!/usr/bin/env python
+# Copyright 2019-2020 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage: ./cleanup_kubernetes_jobs.py [options]
+
+Clean up kubernetes apps that aren't supposed to run on this cluster by deleting them.
+
+Gets the current app list from kubernetes, and then a 'valid_app_list'
+via utils.get_services_for_cluster
+
+If an app in the kubernetes app list isn't in the valid_app_list, it's
+deleted.
+
+Command line options:
+
+- -d <SOA_DIR>, --soa-dir <SOA_DIR>: Specify a SOA config dir to read from
+- -v, --verbose: Verbose output
+- -t <KILL_THRESHOLD>, --kill-threshold: The decimal fraction of apps we think
+    is sane to kill when this job runs
+- -f, --force: Force the killing of apps if we breach the threshold
+- -c, --cluster: Specifies the paasta cluster to check
+- --eks: This flag cleans up only k8 services that shouldn't be running on EKS leaving instances from eks-*.yaml files
+"""
+import argparse
+import logging
+import sys
+import traceback
+from contextlib import contextmanager
+from typing import Dict
+from typing import Generator
+from typing import List
+from typing import Set
+from typing import Tuple
+from typing import Union
+
+from kubernetes.client import V1Deployment
+from kubernetes.client import V1StatefulSet
+from pysensu_yelp import Status
+
+from paasta_tools.eks_tools import EksDeploymentConfig
+from paasta_tools.eks_tools import load_eks_service_config
+from paasta_tools.kubernetes.application.controller_wrappers import DeploymentWrapper
+from paasta_tools.kubernetes.application.controller_wrappers import StatefulSetWrapper
+from paasta_tools.kubernetes.application.tools import Application
+from paasta_tools.kubernetes.application.tools import list_all_applications
+from paasta_tools.kubernetes_tools import KubeClient
+from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig
+from paasta_tools.kubernetes_tools import load_kubernetes_service_config
+from paasta_tools.monitoring_tools import send_event
+from paasta_tools.utils import _log
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import get_services_for_cluster
+from paasta_tools.utils import load_system_paasta_config
+
+log = logging.getLogger(__name__)
+APPLICATION_TYPES = [V1StatefulSet, V1Deployment]
+
+
+class DontKillEverythingError(Exception):
+    pass
+
+
+class StatefulSetsAreNotSupportedError(Exception):
+    pass
+
+
+@contextmanager
+def alert_state_change(application: Application, cluster: str) -> Generator:
+    service = application.kube_deployment.service
+    instance = application.kube_deployment.instance
+    try:
+        yield
+        log_line = (
+            "Deleted stale Kubernetes apps that looks lost: %s"
+            % application.item.metadata.name
+        )
+        _log(
+            service=service,
+            component="deploy",
+            level="event",
+            cluster=cluster,
+            instance=instance,
+            line=log_line,
+        )
+
+    except Exception:
+        loglines = ["Exception raised during cleanup of service %s:" % application]
+        loglines.extend(traceback.format_exc().rstrip().split("\n"))
+        for logline in loglines:
+            _log(
+                service=service,
+                component="deploy",
+                level="debug",
+                cluster=cluster,
+                instance=instance,
+                line=logline,
+            )
+        raise
+
+
+def instance_is_not_bouncing(
+    instance_config: Union[KubernetesDeploymentConfig, EksDeploymentConfig],
+    applications: List[Application],
+) -> bool:
+    """
+
+    :param instance_config: a KubernetesDeploymentConfig or an EksDeploymentConfig with the configuration of the instance
+    :param applications: a list of all deployments or stateful sets on the cluster that match the service
+     and instance of provided instance_config
+    """
+    for application in applications:
+        if isinstance(application, DeploymentWrapper):
+            existing_app = application.item
+            if (
+                (
+                    existing_app.metadata.namespace != instance_config.get_namespace()
+                    and (instance_config.get_bounce_method() == "downthenup")
+                )
+                or (
+                    existing_app.metadata.namespace == instance_config.get_namespace()
+                    and (
+                        instance_config.get_instances()
+                        <= (existing_app.status.ready_replicas or 0)
+                    )
+                )
+            ) or instance_config.get_desired_state() == "stop":
+                return True
+
+        elif (
+            isinstance(application, StatefulSetWrapper)
+            and application.item.metadata.namespace != instance_config.get_namespace()
+        ):
+            log.critical(
+                "Paasta detected a StatefulSet that was migrated to a new namespace"
+                "StatefulSet bouncing across namespaces is not supported"
+            )
+            raise StatefulSetsAreNotSupportedError
+    return False
+
+
+def get_applications_to_kill(
+    applications_dict: Dict[Tuple[str, str], List[Application]],
+    cluster: str,
+    valid_services: Set[Tuple[str, str]],
+    soa_dir: str,
+    eks: bool = False,
+) -> List[Application]:
+    """
+
+    :param applications_dict: A dictionary with (service, instance) as keys and a list of applications for each tuple
+    :param cluster: paasta cluster
+    :param valid_services: a set with the valid (service, instance) tuples for this cluster
+    :param soa_dir: The SOA config directory to read from
+    :return: list of applications to kill
+    """
+    log.info("Determining apps to be killed")
+
+    applications_to_kill: List[Application] = []
+    for (service, instance), applications in applications_dict.items():
+        if len(applications) >= 1:
+            if (service, instance) not in valid_services:
+                applications_to_kill.extend(applications)
+            else:
+                instance_config: Union[KubernetesDeploymentConfig, EksDeploymentConfig]
+                if eks:
+                    instance_config = load_eks_service_config(
+                        cluster=cluster,
+                        service=service,
+                        instance=instance,
+                        soa_dir=soa_dir,
+                    )
+                else:
+                    instance_config = load_kubernetes_service_config(
+                        cluster=cluster,
+                        service=service,
+                        instance=instance,
+                        soa_dir=soa_dir,
+                    )
+                try:
+                    not_bouncing = instance_is_not_bouncing(
+                        instance_config, applications
+                    )
+                except StatefulSetsAreNotSupportedError:
+                    overrides = {
+                        "page": True,
+                        "alert_after": 0,
+                        "tip": f"Revert {service}.{instance} in soa-configs to not include the namespace key.",
+                        "runbook": "y/rb-paasta-namespace",
+                        "ticket": True,
+                    }
+                    send_event(
+                        service=service,
+                        check_name=f"statefulset_bounce_{service}.{instance}",
+                        overrides=overrides,
+                        status=Status.CRITICAL,  # type: ignore
+                        output=f"Unsupported bounce: {service}.{instance}. PaaSTA managed StatefulSets do not support custom namespace",
+                        soa_dir=soa_dir,
+                    )
+                else:
+                    for application in applications:
+                        if (
+                            application.kube_deployment.namespace
+                            != instance_config.get_namespace()
+                            and not_bouncing
+                        ):
+                            applications_to_kill.append(application)
+    return applications_to_kill
+
+
+def cleanup_unused_apps(
+    soa_dir: str,
+    cluster: str,
+    kill_threshold: float = 0.5,
+    force: bool = False,
+    eks: bool = False,
+) -> None:
+    """Clean up old or invalid jobs/apps from kubernetes. Retrieves
+    both a list of apps currently in kubernetes and a list of valid
+    app ids in order to determine what to kill.
+
+    :param soa_dir: The SOA config directory to read from
+    :param cluster: paasta cluster to clean
+    :param kill_threshold: The decimal fraction of apps we think is
+        sane to kill when this job runs.
+    :param force: Force the cleanup if we are above the kill_threshold"""
+    log.info("Creating KubeClient")
+    kube_client = KubeClient()
+
+    log.info("Loading running Kubernetes apps")
+    applications_dict = list_all_applications(kube_client, APPLICATION_TYPES)
+    log.info("Retrieving valid apps from yelpsoa_configs")
+    valid_services = set(
+        get_services_for_cluster(
+            instance_type="eks" if eks else "kubernetes", soa_dir=soa_dir
+        )
+    )
+
+    applications_to_kill: List[Application] = get_applications_to_kill(
+        applications_dict, cluster, valid_services, soa_dir, eks
+    )
+
+    log.debug("Running apps: %s" % list(applications_dict))
+    log.debug("Valid apps: %s" % valid_services)
+    log.debug("Terminating: %s" % applications_to_kill)
+    if applications_to_kill:
+        above_kill_threshold = float(len(applications_to_kill)) / float(
+            len(applications_dict)
+        ) > float(kill_threshold)
+        if above_kill_threshold and not force:
+            log.critical(
+                "Paasta was about to kill more than %s of the running services, this "
+                "is probably a BAD mistake!, run again with --force if you "
+                "really need to destroy everything" % kill_threshold
+            )
+            raise DontKillEverythingError
+
+    for applicaton in applications_to_kill:
+        with alert_state_change(applicaton, cluster):
+            applicaton.deep_delete(kube_client)
+
+
+def parse_args(argv):
+    parser = argparse.ArgumentParser(description="Cleans up stale kubernetes jobs.")
+    parser.add_argument(
+        "-d",
+        "--soa-dir",
+        dest="soa_dir",
+        metavar="SOA_DIR",
+        default=DEFAULT_SOA_DIR,
+        help="define a different soa config directory",
+    )
+    parser.add_argument(
+        "-c",
+        "--cluster",
+        dest="cluster",
+        default=load_system_paasta_config().get_cluster(),
+        help="paasta cluster",
+    )
+    parser.add_argument(
+        "-t",
+        "--kill-threshold",
+        dest="kill_threshold",
+        default=0.5,
+        help="The decimal fraction of apps we think is "
+        "sane to kill when this job runs",
+    )
+    parser.add_argument(
+        "-v", "--verbose", action="store_true", dest="verbose", default=False
+    )
+    parser.add_argument(
+        "-f",
+        "--force",
+        action="store_true",
+        dest="force",
+        default=False,
+        help="Force the cleanup if we are above the " "kill_threshold",
+    )
+    parser.add_argument(
+        "--eks",
+        help="This flag cleans up only k8 services that shouldn't be running on EKS leaving instances from eks-*.yaml files",
+        dest="eks",
+        action="store_true",
+        default=False,
+    )
+    return parser.parse_args(argv)
+
+
+def main(argv=None) -> None:
+    args = parse_args(argv)
+    soa_dir = args.soa_dir
+    kill_threshold = args.kill_threshold
+    force = args.force
+    cluster = args.cluster
+    eks = args.eks
+    if args.verbose:
+        logging.basicConfig(level=logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.WARNING)
+    try:
+        cleanup_unused_apps(
+            soa_dir,
+            cluster=cluster,
+            kill_threshold=kill_threshold,
+            force=force,
+            eks=eks,
+        )
+    except DontKillEverythingError:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

paasta_tools/cleanup_tron_namespaces.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+# Copyright 2015-2018 Yelp Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage: ./cleanup_tron_namespaces.py [options]
+
+Delete namespaces that aren't configured in SOA configs for a Tron cluster.
+
+Gets the list of namespaces from Tron, then compares to the namespaces
+defined in SOA configs.
+
+- -d <SOA_DIR>, --soa-dir <SOA_DIR>: Specify a SOA config dir to read from
+- --dry-run: Print namespaces to be deleted instead of deleting them
+"""
+import argparse
+import sys
+
+from paasta_tools import tron_tools
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Cleans up stale Tron namespaces.")
+    parser.add_argument(
+        "-d",
+        "--soa-dir",
+        dest="soa_dir",
+        metavar="SOA_DIR",
+        default=tron_tools.DEFAULT_SOA_DIR,
+        help="Use a different soa config directory",
+    )
+    parser.add_argument(
+        "--dry-run",
+        dest="dry_run",
+        action="store_true",
+        help="Print namespaces to be deleted, instead of deleting them",
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = parse_args()
+
+    cluster = tron_tools.load_tron_config().get_cluster_name()
+    client = tron_tools.get_tron_client()
+    namespaces = client.list_namespaces()
+    expected_namespaces = tron_tools.get_tron_namespaces(
+        cluster=cluster, soa_dir=args.soa_dir
+    )
+    to_delete = set(namespaces) - set(expected_namespaces) - {"MASTER"}
+
+    if not to_delete:
+        print("No Tron namespaces to remove")
+        sys.exit(0)
+
+    if args.dry_run:
+        print("Dry run, would have removed namespaces:\n  " + "\n  ".join(to_delete))
+        sys.exit(0)
+
+    successes = []
+    errors = []
+    for namespace in to_delete:
+        try:
+            client.update_namespace(namespace, "")
+            successes.append(namespace)
+        except Exception as e:
+            errors.append((namespace, e))
+
+    if successes:
+        print("Successfully removed namespaces:\n", "\n  ".join(successes))
+
+    if errors:
+        print(
+            "Failed to remove namespaces:\n  "
+            + "\n  ".join(
+                [
+                    "{namespace}: {error}".format(namespace=namespace, error=str(error))
+                    for namespace, error in errors
+                ]
+            )
+        )
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

paasta_tools/cli/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

paasta_tools/cli/authentication.py

@@ -0,0 +1,85 @@
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from functools import lru_cache
+
+from botocore.credentials import InstanceMetadataFetcher
+from botocore.credentials import InstanceMetadataProvider
+
+from paasta_tools.utils import load_system_paasta_config
+
+
+try:
+    from vault_tools.paasta_secret import get_client as get_vault_client
+    from vault_tools.paasta_secret import get_vault_url
+    from vault_tools.paasta_secret import get_vault_ca
+    from okta_auth import get_and_cache_jwt_default
+except ImportError:
+
+    def get_vault_client(url: str, capath: str) -> None:
+        pass
+
+    def get_vault_url(ecosystem: str) -> str:
+        return ""
+
+    def get_vault_ca(ecosystem: str) -> str:
+        return ""
+
+    def get_and_cache_jwt_default(client_id: str) -> str:
+        return ""
+
+
+def get_current_ecosystem() -> str:
+    """Get current ecosystem from host configs, defaults to dev if no config is found"""
+    try:
+        with open("/nail/etc/ecosystem") as f:
+            return f.read().strip()
+    except IOError:
+        pass
+    return "devc"
+
+
+@lru_cache(maxsize=1)
+def get_service_auth_token() -> str:
+    """Uses instance profile to authenticate with Vault and generate token for service authentication"""
+    ecosystem = get_current_ecosystem()
+    vault_client = get_vault_client(get_vault_url(ecosystem), get_vault_ca(ecosystem))
+    vault_role = load_system_paasta_config().get_service_auth_vault_role()
+    metadata_provider = InstanceMetadataProvider(
+        iam_role_fetcher=InstanceMetadataFetcher(),
+    )
+    instance_credentials = metadata_provider.load().get_frozen_credentials()
+    vault_client.auth.aws.iam_login(
+        instance_credentials.access_key,
+        instance_credentials.secret_key,
+        instance_credentials.token,
+        mount_point="aws-iam",
+        role=vault_role,
+        use_token=True,
+    )
+    response = vault_client.secrets.identity.generate_signed_id_token(name=vault_role)
+    return response["data"]["token"]
+
+
+def get_sso_auth_token(paasta_apis: bool = False) -> str:
+    """Generate an authentication token for the calling user from the Single Sign On provider
+
+    :param bool paasta_apis: authenticate for PaaSTA APIs
+    """
+    system_config = load_system_paasta_config()
+    client_id = (
+        system_config.get_api_auth_sso_oidc_client_id()
+        if paasta_apis
+        else system_config.get_service_auth_sso_oidc_client_id()
+    )
+    return get_and_cache_jwt_default(client_id)