paasta-tools 1.27.0__py3-none-any.whl → 1.35.8__py3-none-any.whl

{paasta_tools-1.27.0.data → paasta_tools-1.35.8.data}/scripts/setup_kubernetes_job.py

@@ -52,6 +52,7 @@ from paasta_tools.kubernetes_tools import load_kubernetes_service_config_no_cach
 from paasta_tools.metrics import metrics_lib
 from paasta_tools.utils import decompose_job_id
 from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import DeploymentVersion
 from paasta_tools.utils import InvalidJobNameError
 from paasta_tools.utils import load_system_paasta_config
 from paasta_tools.utils import NoConfigurationForServiceError
@@ -241,7 +242,7 @@ def get_kubernetes_deployment_config(
             service_instance_configs_list.append((True, None))
         except NoConfigurationForServiceError:
             error_msg = (
-                f"Could not read kubernetes configuration file for %s.%s in cluster %s"
+                "Could not read kubernetes configuration file for %s.%s in cluster %s"
                 % (service_instance[0], service_instance[1], cluster)
             )
             log.error(error_msg)
@@ -363,6 +364,14 @@ def setup_kube_deployments(
         for deployment in existing_kube_deployments
     }
 
+    existing_deployment_versions: Dict[
+        Tuple[str, str, str], List[DeploymentVersion]
+    ] = {}
+    for deployment in existing_kube_deployments:
+        existing_deployment_versions.setdefault(
+            (deployment.service, deployment.instance, deployment.namespace), []
+        ).append(deployment.deployment_version)
+
     hpa_overrides = hpa_overrides or {}
 
     applications = [
@@ -379,6 +388,39 @@ def setup_kube_deployments(
         else (_, None)
         for _, service_instance in service_instance_configs_list
     ]
+
+    def sort_key(ok_app: Tuple[bool, Optional[Application]]) -> int:
+        """This will return 1 if the desired deployment_version matches an existing deployment_version, 2 if the deployment is unhealthy and 0
+        otherwise. This will cause applications that need a new deployment_version to be handled first.
+        This prioritizes "real" bounces (developers pushing new service versions) over things that are likely to be big
+        bounces (config-only changes).
+        Most of the time, this won't matter much, as we should get through our backlog quickly, but when there is a
+        backlog, we want to avoid blocking developers.
+        """
+        _, app = ok_app
+        if app:
+            if (
+                app.kube_deployment.deployment_version
+                in existing_deployment_versions.get(
+                    (
+                        app.kube_deployment.service,
+                        app.kube_deployment.instance,
+                        app.kube_deployment.namespace,
+                    ),
+                    [],
+                )
+            ):
+                # Desired version exists, so this is just a configuration update; handle this second.
+                return 1
+            else:
+                # Desired version doesn't exist, handle this first.
+                return 0
+        else:
+            # Handle broken app last.
+            return 2
+
+    applications.sort(key=sort_key)
+
     api_updates = 0
     for _, app in applications:
         if app:

{paasta_tools-1.27.0.data → paasta_tools-1.35.8.data}/scripts/setup_prometheus_adapter_config.py

@@ -53,6 +53,9 @@ from paasta_tools.long_running_service_tools import (
 from paasta_tools.long_running_service_tools import (
     DEFAULT_UWSGI_AUTOSCALING_MOVING_AVERAGE_WINDOW,
 )
+from paasta_tools.long_running_service_tools import (
+    DEFAULT_WORKER_LOAD_AUTOSCALING_MOVING_AVERAGE_WINDOW,
+)
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_ACTIVE_REQUESTS
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_CPU
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_GUNICORN
@@ -60,6 +63,7 @@ from paasta_tools.long_running_service_tools import METRICS_PROVIDER_PISCINA
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_PROMQL
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_UWSGI
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_UWSGI_V2
+from paasta_tools.long_running_service_tools import METRICS_PROVIDER_WORKER_LOAD
 from paasta_tools.paasta_service_config_loader import PaastaServiceConfigLoader
 from paasta_tools.utils import DEFAULT_SOA_DIR
 from paasta_tools.utils import get_services_for_cluster
@@ -214,6 +218,10 @@ def create_instance_scaling_rule(
         return create_instance_uwsgi_v2_scaling_rule(
             service, instance_config, metrics_provider_config, paasta_cluster
         )
+    if metrics_provider_config["type"] == METRICS_PROVIDER_WORKER_LOAD:
+        return create_instance_worker_load_scaling_rule(
+            service, instance_config, metrics_provider_config, paasta_cluster
+        )
     if metrics_provider_config["type"] == METRICS_PROVIDER_PISCINA:
         return create_instance_piscina_scaling_rule(
             service, instance_config, metrics_provider_config, paasta_cluster
@@ -523,6 +531,80 @@ def create_instance_uwsgi_v2_scaling_rule(
     }
 
 
+def create_instance_worker_load_scaling_rule(
+    service: str,
+    instance_config: KubernetesDeploymentConfig,
+    metrics_provider_config: MetricsProviderDict,
+    paasta_cluster: str,
+) -> PrometheusAdapterRule:
+    """
+    Creates a Prometheus adapter rule config for a given service instance using generic worker_busy metric.
+    """
+    instance = instance_config.instance
+    moving_average_window = metrics_provider_config.get(
+        "moving_average_window_seconds",
+        DEFAULT_WORKER_LOAD_AUTOSCALING_MOVING_AVERAGE_WINDOW,
+    )
+    deployment_name = get_kubernetes_app_name(service=service, instance=instance)
+
+    # In order for autoscaling to work safely while a service migrates from one namespace to another, the HPA needs to
+    # make sure that the deployment in the new namespace is scaled up enough to handle _all_ the load.
+    # This is because once the new deployment is 100% healthy, cleanup_kubernetes_job will delete the deployment out of
+    # the old namespace all at once, suddenly putting all the load onto the deployment in the new namespace.
+    # To ensure this, we must NOT filter on namespace in worker_filter_terms (which is used when calculating total_load.
+    # This makes sure that desired_instances includes load from all namespaces.
+    worker_filter_terms = f"paasta_cluster='{paasta_cluster}',paasta_service='{service}',paasta_instance='{instance}'"
+
+    # k8s:deployment:pods_status_ready is a metric created by summing kube_pod_status_ready
+    # over paasta service/instance/cluster. it counts the number of ready pods in a paasta
+    # deployment.
+    ready_pods = f"""
+        (sum(
+            k8s:deployment:pods_status_ready{{{worker_filter_terms}}} >= 0
+            or
+            max_over_time(
+                k8s:deployment:pods_status_ready{{{worker_filter_terms}}}[{DEFAULT_EXTRAPOLATION_TIME}s]
+            )
+        ) by (kube_deployment))
+    """
+    load_per_instance = f"""
+        avg(
+            worker_busy{{{worker_filter_terms}}}
+        ) by (kube_pod, kube_deployment)
+    """
+    missing_instances = f"""
+        clamp_min(
+            {ready_pods} - count({load_per_instance}) by (kube_deployment),
+            0
+        )
+    """
+    total_load = f"""
+    (
+        sum(
+            {load_per_instance}
+        ) by (kube_deployment)
+        +
+        {missing_instances}
+    )
+    """
+    total_load_smoothed = f"""
+        avg_over_time(
+            (
+                {total_load}
+            )[{moving_average_window}s:]
+        )
+    """
+
+    metric_name = f"{deployment_name}-worker-load-prom"
+
+    return {
+        "name": {"as": metric_name},
+        "seriesQuery": f"worker_busy{{{worker_filter_terms}}}",
+        "resources": {"template": "kube_<<.Resource>>"},
+        "metricsQuery": _minify_promql(total_load_smoothed),
+    }
+
+
 def create_instance_piscina_scaling_rule(
     service: str,
     instance_config: KubernetesDeploymentConfig,

paasta_tools-1.35.8.dist-info/METADATA

@@ -0,0 +1,79 @@
+Metadata-Version: 2.4
+Name: paasta-tools
+Version: 1.35.8
+Summary: Tools for Yelps SOA infrastructure
+Author: Compute Infrastructure @ Yelp
+Author-email: compute-infra@yelp.com
+Provides: paasta_tools
+Requires-Python: >=3.9.0
+License-File: LICENSE
+Requires-Dist: a-sync>=0.5.0
+Requires-Dist: aiohttp>=3.5.4
+Requires-Dist: argcomplete>=0.8.1
+Requires-Dist: boto
+Requires-Dist: boto3
+Requires-Dist: boto3-type-annotations
+Requires-Dist: botocore
+Requires-Dist: bravado>=10.2.0
+Requires-Dist: certifi
+Requires-Dist: choice>=0.1
+Requires-Dist: containerd
+Requires-Dist: cookiecutter>=1.4.0
+Requires-Dist: croniter
+Requires-Dist: docker
+Requires-Dist: dulwich>=0.17.3
+Requires-Dist: environment-tools
+Requires-Dist: ephemeral-port-reserve>=1.0.1
+Requires-Dist: graphviz
+Requires-Dist: grpcio
+Requires-Dist: gunicorn
+Requires-Dist: humanfriendly
+Requires-Dist: humanize>=0.5.1
+Requires-Dist: inotify>=0.2.8
+Requires-Dist: ipaddress>=1.0.22
+Requires-Dist: isodate>=0.7.2
+Requires-Dist: jsonschema[format]
+Requires-Dist: kazoo>=2.0.0
+Requires-Dist: kubernetes<35.0.0,>=29.0.0
+Requires-Dist: ldap3
+Requires-Dist: manhole
+Requires-Dist: mypy-extensions>=0.3.0
+Requires-Dist: nats-py
+Requires-Dist: nulltype
+Requires-Dist: objgraph
+Requires-Dist: ply
+Requires-Dist: progressbar2>=4.3.2
+Requires-Dist: prometheus-client
+Requires-Dist: pymesos>=0.2.0
+Requires-Dist: pyramid-swagger>=2.3.0
+Requires-Dist: pyramid>=2.0.2
+Requires-Dist: pysensu-yelp>=0.3.4
+Requires-Dist: PyStaticConfiguration
+Requires-Dist: python-crontab>=2.1.1
+Requires-Dist: python-dateutil>=2.4.0
+Requires-Dist: python-iptables
+Requires-Dist: pytimeparse>=1.1.0
+Requires-Dist: pytz>=2014.10
+Requires-Dist: requests>=2.18.4
+Requires-Dist: requests-cache>=0.4.10
+Requires-Dist: retry
+Requires-Dist: ruamel.yaml
+Requires-Dist: sensu-plugin
+Requires-Dist: service-configuration-lib>=3.3.7
+Requires-Dist: signalfx
+Requires-Dist: slackclient>=1.2.1
+Requires-Dist: sticht>=1.1.0
+Requires-Dist: syslogmp
+Requires-Dist: transitions
+Requires-Dist: typing-extensions
+Requires-Dist: tzlocal
+Requires-Dist: urllib3
+Requires-Dist: utaw>=0.2.0
+Requires-Dist: wsgicors
+Dynamic: author
+Dynamic: author-email
+Dynamic: license-file
+Dynamic: provides
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary