paasta-tools 1.21.3__py3-none-any.whl

paasta_tools/bounce_lib.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import math
+from typing import Callable
+from typing import Collection
+from typing import Dict
+from typing import Sequence
+from typing import Set
+
+from mypy_extensions import Arg
+from mypy_extensions import DefaultArg
+from mypy_extensions import TypedDict
+
+
+log = logging.getLogger(__name__)
+log.addHandler(logging.NullHandler())
+logging.getLogger("requests").setLevel(logging.WARNING)
+
+ZK_LOCK_CONNECT_TIMEOUT_S = 10.0  # seconds to wait to connect to zookeeper
+ZK_LOCK_PATH = "/bounce"
+WAIT_CREATE_S = 3
+WAIT_DELETE_S = 5
+
+BounceMethodConfigDict = TypedDict("BounceMethodConfigDict", {"instances": int})
+
+BounceMethodResult = TypedDict(
+    "BounceMethodResult", {"create_app": bool, "tasks_to_drain": Set}
+)
+
+BounceMethod = Callable[
+    [
+        Arg(BounceMethodConfigDict, "new_config"),
+        Arg(bool, "new_app_running"),
+        Arg(Collection, "happy_new_tasks"),
+        Arg(Sequence, "old_non_draining_tasks"),
+        DefaultArg(float, "margin_factor"),
+    ],
+    BounceMethodResult,
+]
+
+
+_bounce_method_funcs: Dict[str, BounceMethod] = {}
+
+
+def register_bounce_method(name: str) -> Callable[[BounceMethod], BounceMethod]:
+    """Returns a decorator that registers that bounce function at a given name
+    so get_bounce_method_func can find it."""
+
+    def outer(bounce_func: BounceMethod):
+        _bounce_method_funcs[name] = bounce_func
+        return bounce_func
+
+    return outer
+
+
+def get_bounce_method_func(name) -> BounceMethod:
+    return _bounce_method_funcs[name]
+
+
+def list_bounce_methods() -> Collection[str]:
+    return _bounce_method_funcs.keys()
+
+
+@register_bounce_method("brutal")
+def brutal_bounce(
+    new_config: BounceMethodConfigDict,
+    new_app_running: bool,
+    happy_new_tasks: Collection,
+    old_non_draining_tasks: Sequence,
+    margin_factor=1.0,
+) -> BounceMethodResult:
+    """Pays no regard to safety. Starts the new app if necessary, and kills any
+    old ones. Mostly meant as an example of the simplest working bounce method,
+    but might be tolerable for some services.
+
+    :param new_config: The configuration dictionary representing the desired new app.
+    :param new_app_running: Whether there is an app in Marathon with the same ID as the new config.
+    :param happy_new_tasks: Set of MarathonTasks belonging to the new application that are considered healthy and up.
+    :param old_non_draining_tasks: A sequence of tasks not belonging to the new version. Tasks should be ordered from
+                                   most desirable to least desirable.
+    :param margin_factor: the multiplication factor used to calculate the number of instances to be drained
+                          when the crossover method is used.
+    :return: A dictionary representing the desired bounce actions and containing the following keys:
+              - create_app: True if we should start the new Marathon app, False otherwise.
+              - tasks_to_drain: a set of task objects which should be drained and killed. May be empty.
+    """
+    return {
+        "create_app": not new_app_running,
+        "tasks_to_drain": set(old_non_draining_tasks),
+    }
+
+
+@register_bounce_method("upthendown")
+def upthendown_bounce(
+    new_config: BounceMethodConfigDict,
+    new_app_running: bool,
+    happy_new_tasks: Collection,
+    old_non_draining_tasks: Sequence,
+    margin_factor=1.0,
+) -> BounceMethodResult:
+    """Starts a new app if necessary; only kills old apps once all the requested tasks for the new version are running.
+
+    See the docstring for brutal_bounce() for parameters and return value.
+    """
+    if new_app_running and len(happy_new_tasks) == new_config["instances"]:
+        return {"create_app": False, "tasks_to_drain": set(old_non_draining_tasks)}
+    else:
+        return {"create_app": not new_app_running, "tasks_to_drain": set()}
+
+
+@register_bounce_method("crossover")
+def crossover_bounce(
+    new_config: BounceMethodConfigDict,
+    new_app_running: bool,
+    happy_new_tasks: Collection,
+    old_non_draining_tasks: Sequence,
+    margin_factor=1.0,
+) -> BounceMethodResult:
+    """Starts a new app if necessary; slowly kills old apps as instances of the new app become happy.
+
+    See the docstring for brutal_bounce() for parameters and return value.
+    """
+
+    assert margin_factor > 0
+    assert margin_factor <= 1
+
+    needed_count = max(
+        int(math.ceil(new_config["instances"] * margin_factor)) - len(happy_new_tasks),
+        0,
+    )
+
+    return {
+        "create_app": not new_app_running,
+        "tasks_to_drain": set(old_non_draining_tasks[needed_count:]),
+    }
+
+
+@register_bounce_method("downthenup")
+def downthenup_bounce(
+    new_config: BounceMethodConfigDict,
+    new_app_running: bool,
+    happy_new_tasks: Collection,
+    old_non_draining_tasks: Sequence,
+    margin_factor=1.0,
+) -> BounceMethodResult:
+    """Stops any old apps and waits for them to die before starting a new one.
+
+    See the docstring for brutal_bounce() for parameters and return value.
+    """
+    return {
+        "create_app": not old_non_draining_tasks and not new_app_running,
+        "tasks_to_drain": set(old_non_draining_tasks),
+    }
+
+
+@register_bounce_method("down")
+def down_bounce(
+    new_config: BounceMethodConfigDict,
+    new_app_running: bool,
+    happy_new_tasks: Collection,
+    old_non_draining_tasks: Sequence,
+    margin_factor=1.0,
+) -> BounceMethodResult:
+    """
+    Stops old apps, doesn't start any new apps.
+    Used for the graceful_app_drain script.
+    """
+    return {"create_app": False, "tasks_to_drain": set(old_non_draining_tasks)}
+
+
+# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4

paasta_tools/broadcast_log_to_services.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import sys
+
+from paasta_tools.kubernetes_tools import get_all_kubernetes_services_running_here
+from paasta_tools.mesos_tools import MesosSlaveConnectionError
+from paasta_tools.tron_tools import tron_jobs_running_here
+from paasta_tools.utils import _log
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import load_system_paasta_config
+
+
+def broadcast_log_all_services_running_here(line: str, soa_dir=DEFAULT_SOA_DIR) -> None:
+    """Log a line of text to paasta logs of all services running on this host.
+
+    :param line: text to log
+    """
+    system_paasta_config = load_system_paasta_config()
+    cluster = system_paasta_config.get_cluster()
+    services = get_all_services_running_here(cluster, soa_dir)
+    for service, instance, _ in services:
+        _log(
+            line=line,
+            service=service,
+            instance=instance,
+            component="monitoring",
+            cluster=cluster,
+        )
+
+
+def get_all_services_running_here(cluster, soa_dir):
+    try:
+        tron_services = tron_jobs_running_here()
+    except MesosSlaveConnectionError:
+        tron_services = []
+
+    try:
+        kubernetes_services = get_all_kubernetes_services_running_here()
+    except Exception:
+        kubernetes_services = []
+
+    return tron_services + kubernetes_services
+
+
+def main() -> None:
+    broadcast_log_all_services_running_here(sys.stdin.read().strip())
+
+
+if __name__ == "__main__":
+    main()

paasta_tools/cassandracluster_tools.py

@@ -0,0 +1,210 @@
+# Copyright 2015-2019 Yelp Inc.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+from typing import List
+from typing import Mapping
+from typing import Optional
+
+import service_configuration_lib
+
+from paasta_tools.kubernetes_tools import sanitise_kubernetes_name
+from paasta_tools.kubernetes_tools import sanitised_cr_name
+from paasta_tools.long_running_service_tools import LongRunningServiceConfig
+from paasta_tools.long_running_service_tools import LongRunningServiceConfigDict
+from paasta_tools.utils import BranchDictV2
+from paasta_tools.utils import compose_job_id
+from paasta_tools.utils import decompose_job_id
+from paasta_tools.utils import deep_merge_dictionaries
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import InvalidJobNameError
+from paasta_tools.utils import load_service_instance_config
+from paasta_tools.utils import load_v2_deployments_json
+
+KUBERNETES_NAMESPACE = "paasta-cassandraclusters"
+
+log = logging.getLogger(__name__)
+log.addHandler(logging.NullHandler())
+
+
+class CassandraClusterDeploymentConfigDict(LongRunningServiceConfigDict, total=False):
+    replicas: int
+
+
+class CassandraClusterDeploymentConfig(LongRunningServiceConfig):
+    config_dict: CassandraClusterDeploymentConfigDict
+
+    config_filename_prefix = "cassandracluster"
+
+    def __init__(
+        self,
+        service: str,
+        cluster: str,
+        instance: str,
+        config_dict: CassandraClusterDeploymentConfigDict,
+        branch_dict: Optional[BranchDictV2],
+        soa_dir: str = DEFAULT_SOA_DIR,
+    ) -> None:
+
+        super().__init__(
+            cluster=cluster,
+            instance=instance,
+            service=service,
+            soa_dir=soa_dir,
+            config_dict=config_dict,
+            branch_dict=branch_dict,
+        )
+
+    def get_service_name_smartstack(self) -> str:
+        """
+        To support apollo we always register in
+        cassandra_<cluster>.main
+        """
+        return "cassandra_" + self.get_instance()
+
+    def get_nerve_namespace(self) -> str:
+        """
+        To support apollo we always register in
+        cassandra_<cluster>.main
+        """
+        return "main"
+
+    def get_registrations(self) -> List[str]:
+        """
+        To support apollo we always register in
+        cassandra_<cluster>.main
+        """
+        registrations = self.config_dict.get("registrations", [])
+        for registration in registrations:
+            try:
+                decompose_job_id(registration)
+            except InvalidJobNameError:
+                log.error(
+                    "Provided registration {} for service "
+                    "{} is invalid".format(registration, self.service)
+                )
+
+        return registrations or [
+            compose_job_id(self.get_service_name_smartstack(), "main")
+        ]
+
+    def get_kubernetes_namespace(self) -> str:
+        return KUBERNETES_NAMESPACE
+
+    def get_namespace(self) -> str:
+        """Get namespace from config, default to 'paasta'"""
+        return self.config_dict.get("namespace", KUBERNETES_NAMESPACE)
+
+    def get_instances(self, with_limit: bool = True) -> int:
+        return self.config_dict.get("replicas", 1)
+
+    def get_bounce_method(self) -> str:
+        """
+        This isn't really true since we use the StatefulSet RollingUpdate strategy
+        However for the paasta-api we need to map to a paasta bounce method and
+        crossover is the closest
+        """
+        return "crossover"
+
+    def get_sanitised_service_name(self) -> str:
+        return sanitise_kubernetes_name(self.get_service())
+
+    def get_sanitised_instance_name(self) -> str:
+        return sanitise_kubernetes_name(self.get_instance())
+
+    def get_sanitised_deployment_name(self) -> str:
+        return self.get_sanitised_instance_name()
+
+    def validate(
+        self,
+        params: List[str] = [
+            "cpus",
+            "security",
+            "dependencies_reference",
+            "deploy_group",
+        ],
+    ) -> List[str]:
+        # Use InstanceConfig to validate shared config keys like cpus and mem
+        # TODO: add mem back to this list once we fix PAASTA-15582 and
+        # move to using the same units as flink/marathon etc.
+        error_msgs = super().validate(params=params)
+
+        if error_msgs:
+            name = self.get_instance()
+            return [f"{name}: {msg}" for msg in error_msgs]
+        else:
+            return []
+
+
+def load_cassandracluster_instance_config(
+    service: str,
+    instance: str,
+    cluster: str,
+    load_deployments: bool = True,
+    soa_dir: str = DEFAULT_SOA_DIR,
+) -> CassandraClusterDeploymentConfig:
+    """Read a service instance's configuration for CassandraCluster.
+
+    If a branch isn't specified for a config, the 'branch' key defaults to
+    paasta-${cluster}.${instance}.
+
+    :param service: The service name
+    :param instance: The instance of the service to retrieve
+    :param cluster: The cluster to read the configuration for
+    :param load_deployments: A boolean indicating if the corresponding deployments.json for this service
+                             should also be loaded
+    :param soa_dir: The SOA configuration directory to read from
+    :returns: A dictionary of whatever was in the config for the service instance"""
+    general_config = service_configuration_lib.read_service_configuration(
+        service, soa_dir=soa_dir
+    )
+    instance_config = load_service_instance_config(
+        service, instance, "cassandracluster", cluster, soa_dir=soa_dir
+    )
+    general_config = deep_merge_dictionaries(
+        overrides=instance_config, defaults=general_config
+    )
+
+    branch_dict: Optional[BranchDictV2] = None
+    if load_deployments:
+        deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
+        temp_instance_config = CassandraClusterDeploymentConfig(
+            service=service,
+            cluster=cluster,
+            instance=instance,
+            config_dict=general_config,
+            branch_dict=None,
+            soa_dir=soa_dir,
+        )
+        branch = temp_instance_config.get_branch()
+        deploy_group = temp_instance_config.get_deploy_group()
+        branch_dict = deployments_json.get_branch_dict(service, branch, deploy_group)
+
+    return CassandraClusterDeploymentConfig(
+        service=service,
+        cluster=cluster,
+        instance=instance,
+        config_dict=general_config,
+        branch_dict=branch_dict,
+        soa_dir=soa_dir,
+    )
+
+
+# TODO: read this from CRD in service configs
+def cr_id(service: str, instance: str) -> Mapping[str, str]:
+    return dict(
+        group="yelp.com",
+        version="v1alpha1",
+        namespace=KUBERNETES_NAMESPACE,
+        plural="cassandraclusters",
+        name=sanitised_cr_name(service, instance),
+    )

paasta_tools/check_autoscaler_max_instances.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python
+import argparse
+import asyncio
+import logging
+from typing import Type
+
+import pysensu_yelp
+
+from paasta_tools.eks_tools import EksDeploymentConfig
+from paasta_tools.instance import kubernetes as pik
+from paasta_tools.kubernetes_tools import get_kubernetes_app_name
+from paasta_tools.kubernetes_tools import KubeClient
+from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig
+from paasta_tools.metrics.metastatus_lib import suffixed_number_value
+from paasta_tools.monitoring_tools import send_event
+from paasta_tools.paasta_service_config_loader import PaastaServiceConfigLoader
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import list_services
+from paasta_tools.utils import load_system_paasta_config
+from paasta_tools.utils import SystemPaastaConfig
+
+log = logging.getLogger(__name__)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description=(
+            "Check all autoscaled services to see if they're at their max_instances. If"
+            " so, send an alert if their utilization is above"
+            " max_instances_alert_threshold."
+        )
+    )
+    parser.add_argument(
+        "-d",
+        "--soa-dir",
+        dest="soa_dir",
+        default=DEFAULT_SOA_DIR,
+        help="Use a different soa config directory",
+    )
+    parser.add_argument(
+        "--dry-run",
+        dest="dry_run",
+        action="store_true",
+        help="Print Sensu alert events instead of sending them",
+    )
+    return parser.parse_args()
+
+
+async def check_max_instances(
+    soa_dir: str,
+    cluster: str,
+    instance_type_class: Type[KubernetesDeploymentConfig],
+    system_paasta_config: SystemPaastaConfig,
+    dry_run: bool = False,
+):
+    kube_client = KubeClient()
+    for service in list_services(soa_dir=soa_dir):
+        service_config = PaastaServiceConfigLoader(service=service, soa_dir=soa_dir)
+        for job_config in service_config.instance_configs(
+            cluster=cluster, instance_type_class=instance_type_class
+        ):
+            instance = job_config.get_instance()
+            if not job_config.get_autoscaling_metric_spec(
+                name=get_kubernetes_app_name(service, instance),
+                cluster=cluster,
+                kube_client=kube_client,
+                namespace=job_config.get_namespace(),
+            ):
+                # Not an instance that uses HPA, don't check.
+                # TODO: should we send status=0 here, in case someone disables autoscaling for their service / changes
+                # to bespoke autoscaler?
+                continue
+
+            if not job_config.get_docker_image():
+                # skip services that haven't been marked for deployment yet.
+                continue
+
+            autoscaling_status = await pik.autoscaling_status(
+                kube_client=kube_client,
+                job_config=job_config,
+                namespace=job_config.get_namespace(),
+            )
+            if autoscaling_status["min_instances"] == -1:
+                log.warning(
+                    f"HPA {job_config.get_sanitised_deployment_name()} not found."
+                )
+                continue
+
+            if (
+                autoscaling_status["min_instances"]
+                == autoscaling_status["max_instances"]
+            ) and "canary" in instance:
+                status = pysensu_yelp.Status.OK
+                output = (
+                    f"Not checking {service}.{instance} as the instance name contains"
+                    ' "canary" and min_instances == max_instances.'
+                )
+            elif (
+                autoscaling_status["desired_replicas"]
+                >= autoscaling_status["max_instances"]
+            ):
+
+                metrics_provider_configs = job_config.get_autoscaling_params()[
+                    "metrics_providers"
+                ]
+
+                status = pysensu_yelp.Status.UNKNOWN
+                output = "how are there no metrics for this thing?"
+
+                # This makes an assumption that the metrics currently used by the HPA are exactly the same order (and
+                # length) as the list of metrics_providers dictionaries. This should generally be true, but between
+                # yelpsoa-configs being pushed and the HPA actually being updated it may not be true. This might cause
+                # spurious alerts, but hopefully the frequency is low. We can add some safeguards if it's a problem.
+                # (E.g. smarter matching between the status dicts and the config dicts, or bailing/not alerting if the
+                # lists aren't the same lengths.)
+                for metric, metrics_provider_config in zip(
+                    autoscaling_status["metrics"], metrics_provider_configs
+                ):
+
+                    setpoint = metrics_provider_config["setpoint"]
+                    threshold = metrics_provider_config.get(
+                        "max_instances_alert_threshold",
+                        setpoint,
+                    )
+
+                    try:
+                        current_value = suffixed_number_value(metric["current_value"])
+                        target_value = suffixed_number_value(metric["target_value"])
+                    except KeyError:
+                        # we likely couldn't find values for the current metric from autoscaling status
+                        # if this is the only metric, we will return UNKNOWN+this error
+                        # suggest fixing their autoscaling config
+                        output = f'{service}.{instance}: Service is at max_instances, and there is an error fetching your {metrics_provider_config["type"]} metric. Check your autoscaling configs or reach out to #paasta.'
+                    else:
+                        # target_value can be 100*setpoint (for cpu), 1 (for uwsgi, piscina, gunicorn,
+                        # active_requests), or setpoint (for promql).
+                        # Here we divide current_value by target_value to find the ratio of utilization to setpoint,
+                        # and then multiply by setpoint to find the actual utilization in the same units as setpoint.
+                        utilization = setpoint * current_value / target_value
+
+                        if threshold == setpoint:
+                            threshold_description = f"setpoint ({threshold})"
+                        else:
+                            threshold_description = (
+                                f"max_instances_alert_threshold ({threshold})"
+                            )
+
+                        if utilization > threshold:
+                            status = pysensu_yelp.Status.CRITICAL
+                            output = (
+                                f"{service}.{instance}: Service is at max_instances, and"
+                                f" utilization ({utilization}) is greater than"
+                                f" {threshold_description}."
+                            )
+                        else:
+                            status = pysensu_yelp.Status.OK
+                            output = (
+                                f"{service}.{instance}: Service is at max_instances, but"
+                                f" utilization ({utilization}) is less than"
+                                f" {threshold_description}."
+                            )
+            else:
+                status = pysensu_yelp.Status.OK
+                output = f"{service}.{instance} is below max_instances."
+
+            monitoring_overrides = job_config.get_monitoring()
+            monitoring_overrides.update(
+                {
+                    "page": False,  # TODO: remove this line once this alert has been deployed for a little while.
+                    "runbook": "y/check-autoscaler-max-instances",
+                    "realert_every": 60,  # The check runs once a minute, so this would realert every hour.
+                    "tip": (
+                        "The autoscaler wants to scale up to handle additional load"
+                        " because your service is overloaded, but cannot scale any"
+                        " higher because of max_instances. You may want to bump"
+                        " max_instances. To make this alert quieter, adjust"
+                        " autoscaling.metrics_providers[n].max_instances_alert_threshold in yelpsoa-configs."
+                    ),
+                }
+            )
+            send_event(
+                service,
+                check_name=f"check_autoscaler_max_instances.{service}.{instance}",
+                overrides=monitoring_overrides,
+                status=status,
+                output=output,
+                soa_dir=soa_dir,
+                ttl=None,
+                cluster=cluster,
+                system_paasta_config=system_paasta_config,
+                dry_run=dry_run,
+            )
+
+
+def main():
+    args = parse_args()
+    system_paasta_config = load_system_paasta_config()
+
+    for instance_type_class in [KubernetesDeploymentConfig, EksDeploymentConfig]:
+        asyncio.run(
+            check_max_instances(
+                soa_dir=args.soa_dir,
+                cluster=system_paasta_config.get_cluster(),
+                instance_type_class=instance_type_class,
+                system_paasta_config=system_paasta_config,
+                dry_run=args.dry_run,
+            )
+        )
+
+
+if __name__ == "__main__":
+    main()