paasta-tools 1.30.13__tar.gz → 1.31.0__tar.gz

{paasta_tools-1.30.13/paasta_tools.egg-info → paasta_tools-1.31.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: paasta-tools
-Version: 1.30.13
+Version: 1.31.0
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com

{paasta_tools-1.30.13 → paasta_tools-1.31.0}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.30.13"
+__version__ = "1.31.0"

{paasta_tools-1.30.13 → paasta_tools-1.31.0}/paasta_tools/cli/schemas/autoscaling_schema.json

@@ -10,8 +10,9 @@
                     "cpu",
                     "piscina",
                     "gunicorn",
-                    "arbitrary_promql",
-                    "active-requests"
+                    "active-requests",
+                    "arbitrary-promql",
+                    "worker-load"
                 ]
             },
             "decision_policy": {

{paasta_tools-1.30.13 → paasta_tools-1.31.0}/paasta_tools/kubernetes_tools.py

@@ -151,6 +151,7 @@ from paasta_tools.long_running_service_tools import METRICS_PROVIDER_PISCINA
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_PROMQL
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_UWSGI
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_UWSGI_V2
+from paasta_tools.long_running_service_tools import METRICS_PROVIDER_WORKER_LOAD
 from paasta_tools.long_running_service_tools import ServiceNamespaceConfig
 from paasta_tools.secret_tools import get_secret_name_from_ref
 from paasta_tools.secret_tools import is_secret_ref
@@ -874,7 +875,10 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
                     ),
                 ),
             )
-        elif provider["type"] == METRICS_PROVIDER_UWSGI_V2:
+        elif provider["type"] in {
+            METRICS_PROVIDER_UWSGI_V2,
+            METRICS_PROVIDER_WORKER_LOAD,
+        }:
             return V2MetricSpec(
                 type="Object",
                 object=V2ObjectMetricSource(
@@ -2445,22 +2449,29 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
 
         # not all services use autoscaling, so we label those that do in order to have
         # prometheus selectively discover/scrape them
-        if self.should_use_metrics_provider(METRICS_PROVIDER_UWSGI):
-            # UWSGI no longer needs a label to indicate it needs to be scraped as all pods are checked for the uwsgi stats port by our centralized uwsgi-exporter
-            # But we do still need deploy_group for relabeling properly
-            # this should probably eventually be made into a default label,
+        metrics_providers_needing_deploy_group = [
+            METRICS_PROVIDER_UWSGI,
+            METRICS_PROVIDER_PISCINA,
+            METRICS_PROVIDER_GUNICORN,
+            METRICS_PROVIDER_WORKER_LOAD,
+        ]
+
+        if any(
+            self.should_use_metrics_provider(provider)
+            for provider in metrics_providers_needing_deploy_group
+        ):
+            # Deploy group is needed for Prometheus relabeling properly
+            # This should probably eventually be made into a default label,
             # but for now we're fine with it being behind these feature toggles.
-            # ideally, we'd also have the docker image here for ease-of-use
+            # Ideally, we'd also have the docker image here for ease-of-use
             # in Prometheus relabeling, but that information is over the
             # character limit for k8s labels (63 chars)
             labels["paasta.yelp.com/deploy_group"] = self.get_deploy_group()
 
         if self.should_use_metrics_provider(METRICS_PROVIDER_PISCINA):
-            labels["paasta.yelp.com/deploy_group"] = self.get_deploy_group()
             labels["paasta.yelp.com/scrape_piscina_prometheus"] = "true"
 
         if self.should_use_metrics_provider(METRICS_PROVIDER_GUNICORN):
-            labels["paasta.yelp.com/deploy_group"] = self.get_deploy_group()
             labels["paasta.yelp.com/scrape_gunicorn_prometheus"] = "true"
 
         # the default AWS LB Controller behavior is to enable this by-namespace

{paasta_tools-1.30.13 → paasta_tools-1.31.0}/paasta_tools/long_running_service_tools.py

@@ -41,6 +41,7 @@ DEFAULT_ACTIVE_REQUESTS_AUTOSCALING_MOVING_AVERAGE_WINDOW = 1800
 DEFAULT_UWSGI_AUTOSCALING_MOVING_AVERAGE_WINDOW = 1800
 DEFAULT_PISCINA_AUTOSCALING_MOVING_AVERAGE_WINDOW = 1800
 DEFAULT_GUNICORN_AUTOSCALING_MOVING_AVERAGE_WINDOW = 1800
+DEFAULT_WORKER_LOAD_AUTOSCALING_MOVING_AVERAGE_WINDOW = 1800
 
 METRICS_PROVIDER_CPU = "cpu"
 METRICS_PROVIDER_UWSGI = "uwsgi"
@@ -48,7 +49,8 @@ METRICS_PROVIDER_UWSGI_V2 = "uwsgi-v2"
 METRICS_PROVIDER_GUNICORN = "gunicorn"
 METRICS_PROVIDER_PISCINA = "piscina"
 METRICS_PROVIDER_ACTIVE_REQUESTS = "active-requests"
-METRICS_PROVIDER_PROMQL = "arbitrary_promql"
+METRICS_PROVIDER_PROMQL = "arbitrary-promql"
+METRICS_PROVIDER_WORKER_LOAD = "worker-load"
 
 ALL_METRICS_PROVIDERS = [
     METRICS_PROVIDER_CPU,
@@ -58,6 +60,7 @@ ALL_METRICS_PROVIDERS = [
     METRICS_PROVIDER_PISCINA,
     METRICS_PROVIDER_ACTIVE_REQUESTS,
     METRICS_PROVIDER_PROMQL,
+    METRICS_PROVIDER_WORKER_LOAD,
 ]
 
 

{paasta_tools-1.30.13 → paasta_tools-1.31.0}/paasta_tools/setup_prometheus_adapter_config.py

@@ -53,6 +53,9 @@ from paasta_tools.long_running_service_tools import (
 from paasta_tools.long_running_service_tools import (
     DEFAULT_UWSGI_AUTOSCALING_MOVING_AVERAGE_WINDOW,
 )
+from paasta_tools.long_running_service_tools import (
+    DEFAULT_WORKER_LOAD_AUTOSCALING_MOVING_AVERAGE_WINDOW,
+)
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_ACTIVE_REQUESTS
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_CPU
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_GUNICORN
@@ -60,6 +63,7 @@ from paasta_tools.long_running_service_tools import METRICS_PROVIDER_PISCINA
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_PROMQL
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_UWSGI
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_UWSGI_V2
+from paasta_tools.long_running_service_tools import METRICS_PROVIDER_WORKER_LOAD
 from paasta_tools.paasta_service_config_loader import PaastaServiceConfigLoader
 from paasta_tools.utils import DEFAULT_SOA_DIR
 from paasta_tools.utils import get_services_for_cluster
@@ -214,6 +218,10 @@ def create_instance_scaling_rule(
         return create_instance_uwsgi_v2_scaling_rule(
             service, instance_config, metrics_provider_config, paasta_cluster
         )
+    if metrics_provider_config["type"] == METRICS_PROVIDER_WORKER_LOAD:
+        return create_instance_worker_load_scaling_rule(
+            service, instance_config, metrics_provider_config, paasta_cluster
+        )
     if metrics_provider_config["type"] == METRICS_PROVIDER_PISCINA:
         return create_instance_piscina_scaling_rule(
             service, instance_config, metrics_provider_config, paasta_cluster
@@ -523,6 +531,80 @@ def create_instance_uwsgi_v2_scaling_rule(
     }
 
 
+def create_instance_worker_load_scaling_rule(
+    service: str,
+    instance_config: KubernetesDeploymentConfig,
+    metrics_provider_config: MetricsProviderDict,
+    paasta_cluster: str,
+) -> PrometheusAdapterRule:
+    """
+    Creates a Prometheus adapter rule config for a given service instance using generic worker_busy metric.
+    """
+    instance = instance_config.instance
+    moving_average_window = metrics_provider_config.get(
+        "moving_average_window_seconds",
+        DEFAULT_WORKER_LOAD_AUTOSCALING_MOVING_AVERAGE_WINDOW,
+    )
+    deployment_name = get_kubernetes_app_name(service=service, instance=instance)
+
+    # In order for autoscaling to work safely while a service migrates from one namespace to another, the HPA needs to
+    # make sure that the deployment in the new namespace is scaled up enough to handle _all_ the load.
+    # This is because once the new deployment is 100% healthy, cleanup_kubernetes_job will delete the deployment out of
+    # the old namespace all at once, suddenly putting all the load onto the deployment in the new namespace.
+    # To ensure this, we must NOT filter on namespace in worker_filter_terms (which is used when calculating total_load.
+    # This makes sure that desired_instances includes load from all namespaces.
+    worker_filter_terms = f"paasta_cluster='{paasta_cluster}',paasta_service='{service}',paasta_instance='{instance}'"
+
+    # k8s:deployment:pods_status_ready is a metric created by summing kube_pod_status_ready
+    # over paasta service/instance/cluster. it counts the number of ready pods in a paasta
+    # deployment.
+    ready_pods = f"""
+        (sum(
+            k8s:deployment:pods_status_ready{{{worker_filter_terms}}} >= 0
+            or
+            max_over_time(
+                k8s:deployment:pods_status_ready{{{worker_filter_terms}}}[{DEFAULT_EXTRAPOLATION_TIME}s]
+            )
+        ) by (kube_deployment))
+    """
+    load_per_instance = f"""
+        avg(
+            worker_busy{{{worker_filter_terms}}}
+        ) by (kube_pod, kube_deployment)
+    """
+    missing_instances = f"""
+        clamp_min(
+            {ready_pods} - count({load_per_instance}) by (kube_deployment),
+            0
+        )
+    """
+    total_load = f"""
+    (
+        sum(
+            {load_per_instance}
+        ) by (kube_deployment)
+        +
+        {missing_instances}
+    )
+    """
+    total_load_smoothed = f"""
+        avg_over_time(
+            (
+                {total_load}
+            )[{moving_average_window}s:]
+        )
+    """
+
+    metric_name = f"{deployment_name}-worker-load-prom"
+
+    return {
+        "name": {"as": metric_name},
+        "seriesQuery": f"worker_busy{{{worker_filter_terms}}}",
+        "resources": {"template": "kube_<<.Resource>>"},
+        "metricsQuery": _minify_promql(total_load_smoothed),
+    }
+
+
 def create_instance_piscina_scaling_rule(
     service: str,
     instance_config: KubernetesDeploymentConfig,

{paasta_tools-1.30.13 → paasta_tools-1.31.0/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: paasta-tools
-Version: 1.30.13
+Version: 1.31.0
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com

{paasta_tools-1.30.13 → paasta_tools-1.31.0}/tests/test_kubernetes_tools.py

@@ -157,6 +157,7 @@ from paasta_tools.long_running_service_tools import METRICS_PROVIDER_GUNICORN
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_PISCINA
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_UWSGI
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_UWSGI_V2
+from paasta_tools.long_running_service_tools import METRICS_PROVIDER_WORKER_LOAD
 from paasta_tools.long_running_service_tools import ServiceNamespaceConfig
 from paasta_tools.secret_tools import SHARED_SECRET_SERVICE
 from paasta_tools.utils import AwsEbsVolume
@@ -1739,6 +1740,7 @@ class TestKubernetesDeploymentConfig:
             METRICS_PROVIDER_UWSGI,
             METRICS_PROVIDER_PISCINA,
             METRICS_PROVIDER_GUNICORN,
+            METRICS_PROVIDER_WORKER_LOAD,
         ],
     )
     @pytest.mark.parametrize(
@@ -1871,7 +1873,10 @@ class TestKubernetesDeploymentConfig:
 
         if autoscaling_metric_provider:
             expected_labels["paasta.yelp.com/deploy_group"] = "fake_group"
-            if autoscaling_metric_provider != METRICS_PROVIDER_UWSGI:
+            if autoscaling_metric_provider in (
+                METRICS_PROVIDER_PISCINA,
+                METRICS_PROVIDER_GUNICORN,
+            ):
                 expected_labels[
                     f"paasta.yelp.com/scrape_{autoscaling_metric_provider}_prometheus"
                 ] = "true"
@@ -2866,6 +2871,87 @@ class TestKubernetesDeploymentConfig:
 
         assert expected_res == return_value
 
+    @mock.patch(
+        "paasta_tools.kubernetes_tools.load_system_paasta_config",
+        autospec=True,
+    )
+    def test_get_autoscaling_metric_spec_worker_load_prometheus(
+        self, fake_system_paasta_config
+    ):
+        config_dict = KubernetesDeploymentConfigDict(
+            {
+                "min_instances": 1,
+                "max_instances": 3,
+                "autoscaling": {
+                    "metrics_providers": [
+                        {
+                            "type": METRICS_PROVIDER_WORKER_LOAD,
+                            "setpoint": 0.5,
+                            "forecast_policy": "moving_average",
+                            "moving_average_window_seconds": 300,
+                        }
+                    ]
+                },
+            }
+        )
+        mock_config = KubernetesDeploymentConfig(  # type: ignore
+            service="service",
+            cluster="cluster",
+            instance="instance",
+            config_dict=config_dict,
+            branch_dict=None,
+        )
+        return_value = KubernetesDeploymentConfig.get_autoscaling_metric_spec(
+            mock_config,
+            "fake_name",
+            "cluster",
+            KubeClient(),
+            "paasta",
+        )
+        expected_res = V2HorizontalPodAutoscaler(
+            kind="HorizontalPodAutoscaler",
+            metadata=V1ObjectMeta(
+                name="fake_name",
+                namespace="paasta",
+                annotations={},
+                labels=mock.ANY,
+            ),
+            spec=V2HorizontalPodAutoscalerSpec(
+                behavior=mock_config.get_autoscaling_scaling_policy(
+                    autoscaling_params={},
+                    max_replicas=3,
+                ),
+                max_replicas=3,
+                min_replicas=1,
+                metrics=[
+                    V2MetricSpec(
+                        type="Object",
+                        object=V2ObjectMetricSource(
+                            metric=V2MetricIdentifier(
+                                name="service-instance-worker-load-prom",
+                            ),
+                            target=V2MetricTarget(
+                                type="AverageValue",
+                                average_value=0.5,
+                            ),
+                            described_object=V2CrossVersionObjectReference(
+                                api_version="apps/v1",
+                                kind="Deployment",
+                                name="fake_name",
+                            ),
+                        ),
+                    ),
+                ],
+                scale_target_ref=V2CrossVersionObjectReference(
+                    api_version="apps/v1",
+                    kind="Deployment",
+                    name="fake_name",
+                ),
+            ),
+        )
+
+        assert expected_res == return_value
+
     @mock.patch(
         "paasta_tools.kubernetes_tools.load_system_paasta_config",
         autospec=True,

{paasta_tools-1.30.13 → paasta_tools-1.31.0}/tests/test_setup_prometheus_adapter_config.py

@@ -8,6 +8,7 @@ from paasta_tools.long_running_service_tools import METRICS_PROVIDER_CPU
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_GUNICORN
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_UWSGI
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_UWSGI_V2
+from paasta_tools.long_running_service_tools import METRICS_PROVIDER_WORKER_LOAD
 from paasta_tools.setup_prometheus_adapter_config import _minify_promql
 from paasta_tools.setup_prometheus_adapter_config import (
     create_instance_active_requests_scaling_rule,
@@ -24,6 +25,9 @@ from paasta_tools.setup_prometheus_adapter_config import (
 from paasta_tools.setup_prometheus_adapter_config import (
     create_instance_uwsgi_v2_scaling_rule,
 )
+from paasta_tools.setup_prometheus_adapter_config import (
+    create_instance_worker_load_scaling_rule,
+)
 from paasta_tools.setup_prometheus_adapter_config import get_rules_for_service_instance
 from paasta_tools.utils import SystemPaastaConfig
 
@@ -155,6 +159,49 @@ def test_create_instance_uwsgi_v2_scaling_rule() -> None:
     )
 
 
+def test_create_instance_worker_load_scaling_rule() -> None:
+    service_name = "test_service"
+    instance_config = mock.Mock(instance="test_instance")
+    metrics_provider_config = MetricsProviderDict(
+        {
+            "type": METRICS_PROVIDER_WORKER_LOAD,
+            "setpoint": 0.1234567890,
+            "moving_average_window_seconds": 20120302,
+        }
+    )
+    paasta_cluster = "test_cluster"
+    rule = create_instance_worker_load_scaling_rule(
+        service=service_name,
+        instance_config=instance_config,
+        metrics_provider_config=metrics_provider_config,
+        paasta_cluster=paasta_cluster,
+    )
+
+    # we test that the format of the dictionary is as expected with mypy
+    # and we don't want to test the full contents of the retval since then
+    # we're basically just writing a change-detector test - instead, we test
+    # that we're actually using our inputs
+    assert service_name in rule["seriesQuery"]
+    assert instance_config.instance in rule["seriesQuery"]
+    assert paasta_cluster in rule["seriesQuery"]
+
+    # Like uwsgi_v2, we don't use the setpoint in this query -- the HPA will have the setpoint as its target.
+    assert str(metrics_provider_config["setpoint"]) not in rule["metricsQuery"]
+    assert (
+        str(metrics_provider_config["moving_average_window_seconds"])
+        in rule["metricsQuery"]
+    )
+
+    # Verify that we're using the generic worker_busy metric instead of uwsgi_worker_busy
+    assert "worker_busy" in rule["seriesQuery"]
+    assert "worker_busy" in rule["metricsQuery"]
+    assert "uwsgi_worker_busy" not in rule["seriesQuery"]
+    assert "uwsgi_worker_busy" not in rule["metricsQuery"]
+
+    # Verify the metric name uses worker-load-prom suffix
+    assert rule["name"]["as"].endswith("-worker-load-prom")
+
+
 def test_create_instance_gunicorn_scaling_rule() -> None:
     service_name = "test_service"
     instance_config = mock.Mock(instance="test_instance")
@@ -279,6 +326,24 @@ def test_create_instance_gunicorn_scaling_rule() -> None:
             ),
             2,
         ),
+        (
+            mock.Mock(
+                instance="instance",
+                get_namespace=mock.Mock(return_value="test_namespace"),
+                get_autoscaling_metrics_provider=mock.Mock(
+                    side_effect=lambda x: (
+                        {
+                            "type": METRICS_PROVIDER_WORKER_LOAD,
+                            "setpoint": 0.1234567890,
+                            "moving_average_window_seconds": 20120302,
+                        }
+                        if x == METRICS_PROVIDER_WORKER_LOAD
+                        else None
+                    )
+                ),
+            ),
+            1,
+        ),
     ],
 )
 def test_get_rules_for_service_instance(