paasta-tools 1.31.2__tar.gz → 1.31.4__tar.gz

{paasta_tools-1.31.2/paasta_tools.egg-info → paasta_tools-1.31.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: paasta-tools
-Version: 1.31.2
+Version: 1.31.4
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com

{paasta_tools-1.31.2 → paasta_tools-1.31.4}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.31.2"
+__version__ = "1.31.4"

{paasta_tools-1.31.2 → paasta_tools-1.31.4}/paasta_tools/cli/cmds/status.py

@@ -917,11 +917,13 @@ def _print_flink_status_from_job_manager(
             else f"{pod_evicted_count}"
         )
 
+    pods_total_count = pod_running_count + pod_evicted_count + pod_other_count
     output.append(
         "    Pods:"
         f" {pod_running_count} running,"
         f" {evicted} evicted,"
-        f" {pod_other_count} other"
+        f" {pod_other_count} other,"
+        f" {pods_total_count} total"
     )
 
     if not should_job_info_be_shown(status["state"]):
@@ -944,12 +946,19 @@ def _print_flink_status_from_job_manager(
             output.append(str(e))
             return 1
 
+        jobs_total_count = (
+            overview.jobs_running
+            + overview.jobs_finished
+            + overview.jobs_failed
+            + overview.jobs_cancelled
+        )
         output.append(
             "    Jobs:"
             f" {overview.jobs_running} running,"
             f" {overview.jobs_finished} finished,"
             f" {overview.jobs_failed} failed,"
-            f" {overview.jobs_cancelled} cancelled"
+            f" {overview.jobs_cancelled} cancelled,"
+            f" {jobs_total_count} total"
         )
         output.append(
             "   "
@@ -1313,7 +1322,9 @@ def get_version_table_entry(
             for state in ReplicaState
             if state in replica_state_counts
         ]
-        entry.append(f"  Replica States: {' / '.join(replica_state_display)}")
+        entry.append(
+            f"  Replica States: {' / '.join(replica_state_display)} / {replica_state_counts.total()} total"
+        )
         if not verbose:
             unhealthy_replicas = [
                 (state, pod) for state, pod in replica_states if state.is_unhealthy()

{paasta_tools-1.31.2 → paasta_tools-1.31.4}/paasta_tools/instance/kubernetes.py

@@ -1,4 +1,5 @@
 import asyncio
+import logging
 from asyncio.tasks import Task
 from collections import defaultdict
 from enum import Enum
@@ -17,6 +18,7 @@ from typing import Union
 
 import a_sync
 import pytz
+import requests.exceptions
 from kubernetes.client import V1Container
 from kubernetes.client import V1ControllerRevision
 from kubernetes.client import V1Pod
@@ -75,6 +77,8 @@ INSTANCE_TYPE_CR_ID = dict(
     monkrelaycluster=monkrelaycluster_tools.cr_id,
 )
 
+logger = logging.getLogger(__name__)
+
 
 class ServiceMesh(Enum):
     SMARTSTACK = "smartstack"
@@ -348,31 +352,49 @@ async def mesh_status(
 
     pods = await pods_task
     for location, hosts in node_hostname_by_location.items():
-        host = replication_checker.get_hostname_in_pool(hosts, instance_pool)
-        if service_mesh == ServiceMesh.SMARTSTACK:
-            mesh_status["locations"].append(
-                _build_smartstack_location_dict(
-                    synapse_host=host,
-                    synapse_port=settings.system_paasta_config.get_synapse_port(),
-                    synapse_haproxy_url_format=settings.system_paasta_config.get_synapse_haproxy_url_format(),
-                    registration=registration,
-                    pods=pods,
-                    location=location,
-                    should_return_individual_backends=should_return_individual_backends,
-                )
-            )
-        elif service_mesh == ServiceMesh.ENVOY:
-            mesh_status["locations"].append(
-                _build_envoy_location_dict(
-                    envoy_host=host,
-                    envoy_admin_port=settings.system_paasta_config.get_envoy_admin_port(),
-                    envoy_admin_endpoint_format=settings.system_paasta_config.get_envoy_admin_endpoint_format(),
-                    registration=registration,
-                    pods=pods,
-                    location=location,
-                    should_return_individual_backends=should_return_individual_backends,
-                )
-            )
+        max_retries = 3
+
+        for attempt in range(max_retries):
+            host = replication_checker.get_hostname_in_pool(hosts, instance_pool)
+            try:
+                if service_mesh == ServiceMesh.SMARTSTACK:
+                    location_dict = _build_smartstack_location_dict(
+                        synapse_host=host,
+                        synapse_port=settings.system_paasta_config.get_synapse_port(),
+                        synapse_haproxy_url_format=settings.system_paasta_config.get_synapse_haproxy_url_format(),
+                        registration=registration,
+                        pods=pods,
+                        location=location,
+                        should_return_individual_backends=should_return_individual_backends,
+                    )
+                elif service_mesh == ServiceMesh.ENVOY:
+                    location_dict = _build_envoy_location_dict(
+                        envoy_host=host,
+                        envoy_admin_port=settings.system_paasta_config.get_envoy_admin_port(),
+                        envoy_admin_endpoint_format=settings.system_paasta_config.get_envoy_admin_endpoint_format(),
+                        registration=registration,
+                        pods=pods,
+                        location=location,
+                        should_return_individual_backends=should_return_individual_backends,
+                    )
+
+                mesh_status["locations"].append(location_dict)
+                return mesh_status
+
+            except requests.exceptions.ConnectTimeout:
+                if attempt < max_retries - 1:
+                    logger.warning(
+                        "attempt %s/%s: Unable to connect to %s, retrying (on another host, hopefully)...",
+                        attempt,
+                        max_retries,
+                        host,
+                    )
+                    continue
+                else:
+                    logger.critical(
+                        "Unable to connect to %s, not retrying again.", host
+                    )
+                    raise
     return mesh_status
 
 

{paasta_tools-1.31.2 → paasta_tools-1.31.4}/paasta_tools/kubernetes_tools.py

@@ -2471,7 +2471,9 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
         if self.should_use_metrics_provider(METRICS_PROVIDER_PISCINA):
             labels["paasta.yelp.com/scrape_piscina_prometheus"] = "true"
 
-        if self.should_use_metrics_provider(METRICS_PROVIDER_GUNICORN):
+        if self.should_use_metrics_provider(
+            METRICS_PROVIDER_GUNICORN
+        ) or self.should_use_metrics_provider(METRICS_PROVIDER_WORKER_LOAD):
             labels["paasta.yelp.com/scrape_gunicorn_prometheus"] = "true"
 
         # the default AWS LB Controller behavior is to enable this by-namespace

{paasta_tools-1.31.2 → paasta_tools-1.31.4/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: paasta-tools
-Version: 1.31.2
+Version: 1.31.4
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com

{paasta_tools-1.31.2 → paasta_tools-1.31.4}/tests/test_kubernetes_tools.py

@@ -1880,6 +1880,8 @@ class TestKubernetesDeploymentConfig:
                 expected_labels[
                     f"paasta.yelp.com/scrape_{autoscaling_metric_provider}_prometheus"
                 ] = "true"
+            elif autoscaling_metric_provider == METRICS_PROVIDER_WORKER_LOAD:
+                expected_labels["paasta.yelp.com/scrape_gunicorn_prometheus"] = "true"
         if autoscaling_metric_provider in (
             METRICS_PROVIDER_UWSGI,
             METRICS_PROVIDER_GUNICORN,