paasta-tools 1.30.7__py3-none-any.whl → 1.30.9__py3-none-any.whl

paasta_tools/metrics/metastatus_lib.py

@@ -43,7 +43,6 @@ from paasta_tools.kubernetes_tools import paasta_prefixed
 from paasta_tools.kubernetes_tools import PodStatus
 from paasta_tools.mesos.master import MesosMetrics
 from paasta_tools.mesos.master import MesosState
-from paasta_tools.mesos_maintenance import MAINTENANCE_ROLE
 from paasta_tools.mesos_tools import get_all_tasks_from_state
 from paasta_tools.mesos_tools import get_mesos_quorum
 from paasta_tools.mesos_tools import get_number_of_mesos_masters
@@ -97,9 +96,6 @@ def get_mesos_cpu_status(
     total = metrics["master/cpus_total"]
     used = metrics["master/cpus_used"]
 
-    for slave in mesos_state["slaves"]:
-        used += reserved_maintenence_resources(slave["reserved_resources"])["cpus"]
-
     available = total - used
     return total, used, available
 
@@ -135,9 +131,6 @@ def get_mesos_memory_status(
     total = metrics["master/mem_total"]
     used = metrics["master/mem_used"]
 
-    for slave in mesos_state["slaves"]:
-        used += reserved_maintenence_resources(slave["reserved_resources"])["mem"]
-
     available = total - used
 
     return total, used, available
@@ -176,9 +169,6 @@ def get_mesos_disk_status(
     total = metrics["master/disk_total"]
     used = metrics["master/disk_used"]
 
-    for slave in mesos_state["slaves"]:
-        used += reserved_maintenence_resources(slave["reserved_resources"])["disk"]
-
     available = total - used
     return total, used, available
 
@@ -216,9 +206,6 @@ def get_mesos_gpu_status(
     total = metrics["master/gpus_total"]
     used = metrics["master/gpus_used"]
 
-    for slave in mesos_state["slaves"]:
-        used += reserved_maintenence_resources(slave["reserved_resources"])["gpus"]
-
     available = total - used
     return total, used, available
 
@@ -647,11 +634,6 @@ def calculate_resource_utilization_for_slaves(
     for task in tasks:
         task_resources = task["resources"]
         resource_free_dict.subtract(Counter(filter_mesos_state_metrics(task_resources)))
-    for slave in slaves:
-        filtered_resources = filter_mesos_state_metrics(
-            reserved_maintenence_resources(slave["reserved_resources"])
-        )
-        resource_free_dict.subtract(Counter(filtered_resources))
     return {
         "free": ResourceInfo(
             cpus=resource_free_dict["cpus"],
@@ -1102,9 +1084,3 @@ def get_table_rows_for_resource_info_dict(
     return attribute_values + format_row_for_resource_utilization_healthchecks(
         healthcheck_utilization_pairs
     )
-
-
-def reserved_maintenence_resources(
-    resources: MesosResources,
-):
-    return resources.get(MAINTENANCE_ROLE, {"cpus": 0, "mem": 0, "disk": 0, "gpus": 0})

paasta_tools/metrics/metrics_lib.py

@@ -8,6 +8,7 @@ from inspect import currentframe
 from types import TracebackType
 from typing import Any
 from typing import Callable
+from typing import cast
 from typing import Dict
 from typing import Optional
 from typing import Type
@@ -110,10 +111,18 @@ class MeteoriteMetrics(BaseMetrics):
             )
 
     def create_timer(self, name: str, **kwargs: Any) -> TimerProtocol:
-        return yelp_meteorite.create_timer(self.base_name + "." + name, **kwargs)
+        # yelp_meteorite returns an EmptyMetric object if the timer is misconfigured
+        # ...but that doesn't have the same interface ;_;
+        return cast(
+            Timer, yelp_meteorite.create_timer(self.base_name + "." + name, **kwargs)
+        )
 
     def create_gauge(self, name: str, **kwargs: Any) -> GaugeProtocol:
-        return yelp_meteorite.create_gauge(self.base_name + "." + name, **kwargs)
+        # yelp_meteorite returns an EmptyMetric object if the gauge is misconfigured
+        # ...but that doesn't have the same interface ;_;
+        return cast(
+            Gauge, yelp_meteorite.create_gauge(self.base_name + "." + name, **kwargs)
+        )
 
     def create_counter(self, name: str, **kwargs: Any) -> CounterProtocol:
         return yelp_meteorite.create_counter(self.base_name + "." + name, **kwargs)
@@ -163,7 +172,7 @@ class Gauge(GaugeProtocol):
         log.debug(f"gauge {self.name} set to {value}")
 
 
-class Counter(GaugeProtocol):
+class Counter(CounterProtocol):
     def __init__(self, name: str) -> None:
         self.name = name
         self.counter = 0

paasta_tools/setup_kubernetes_job.py

@@ -52,6 +52,7 @@ from paasta_tools.kubernetes_tools import load_kubernetes_service_config_no_cach
 from paasta_tools.metrics import metrics_lib
 from paasta_tools.utils import decompose_job_id
 from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import DeploymentVersion
 from paasta_tools.utils import InvalidJobNameError
 from paasta_tools.utils import load_system_paasta_config
 from paasta_tools.utils import NoConfigurationForServiceError
@@ -241,7 +242,7 @@ def get_kubernetes_deployment_config(
             service_instance_configs_list.append((True, None))
         except NoConfigurationForServiceError:
             error_msg = (
-                f"Could not read kubernetes configuration file for %s.%s in cluster %s"
+                "Could not read kubernetes configuration file for %s.%s in cluster %s"
                 % (service_instance[0], service_instance[1], cluster)
             )
             log.error(error_msg)
@@ -363,6 +364,14 @@ def setup_kube_deployments(
         for deployment in existing_kube_deployments
     }
 
+    existing_deployment_versions: Dict[
+        Tuple[str, str, str], List[DeploymentVersion]
+    ] = {}
+    for deployment in existing_kube_deployments:
+        existing_deployment_versions.setdefault(
+            (deployment.service, deployment.instance, deployment.namespace), []
+        ).append(deployment.deployment_version)
+
     hpa_overrides = hpa_overrides or {}
 
     applications = [
@@ -379,6 +388,39 @@ def setup_kube_deployments(
         else (_, None)
         for _, service_instance in service_instance_configs_list
     ]
+
+    def sort_key(ok_app: Tuple[bool, Optional[Application]]) -> int:
+        """This will return 1 if the desired deployment_version matches an existing deployment_version, 2 if the deployment is unhealthy and 0
+        otherwise. This will cause applications that need a new deployment_version to be handled first.
+        This prioritizes "real" bounces (developers pushing new service versions) over things that are likely to be big
+        bounces (config-only changes).
+        Most of the time, this won't matter much, as we should get through our backlog quickly, but when there is a
+        backlog, we want to avoid blocking developers.
+        """
+        _, app = ok_app
+        if app:
+            if (
+                app.kube_deployment.deployment_version
+                in existing_deployment_versions.get(
+                    (
+                        app.kube_deployment.service,
+                        app.kube_deployment.instance,
+                        app.kube_deployment.namespace,
+                    ),
+                    [],
+                )
+            ):
+                # Desired version exists, so this is just a configuration update; handle this second.
+                return 1
+            else:
+                # Desired version doesn't exist, handle this first.
+                return 0
+        else:
+            # Handle broken app last.
+            return 2
+
+    applications.sort(key=sort_key)
+
     api_updates = 0
     for _, app in applications:
         if app:

paasta_tools/setup_tron_namespace.py

@@ -114,7 +114,7 @@ def ensure_service_accounts(job_configs: List[TronJobConfig]) -> None:
                     )
 
 
-def main():
+def main() -> None:
     args = parse_args()
     log_level = logging.DEBUG if args.verbose else logging.INFO
     logging.basicConfig(level=log_level)
@@ -232,7 +232,7 @@ def main():
             log.debug(f"Updated {updated}")
 
         if updated_namespaces != new_configs.keys():
-            skipped = set(new_configs.keys()) - set(updated_namespaces.keys())
+            skipped = list(set(new_configs.keys()) - set(updated_namespaces.keys()))
             log.debug(f"Skipped {skipped}")
 
     skipped_report = skipped if args.verbose else len(skipped)

paasta_tools/tron_tools.py

@@ -1289,7 +1289,7 @@ def load_tron_service_config_no_cache(
 def create_complete_master_config(cluster, soa_dir=DEFAULT_SOA_DIR):
     system_paasta_config = load_system_paasta_config()
     tronfig_folder = get_tronfig_folder(soa_dir=soa_dir, cluster=cluster)
-    config = read_yaml_file(os.path.join(tronfig_folder, f"MASTER.yaml"))
+    config = read_yaml_file(os.path.join(tronfig_folder, "MASTER.yaml"))
     master_config = format_master_config(
         config,
         system_paasta_config.get_volumes(),

paasta_tools/utils.py

@@ -53,6 +53,7 @@ from typing import Collection
 from typing import ContextManager
 from typing import Dict
 from typing import FrozenSet
+from typing import Generic
 from typing import IO
 from typing import Iterable
 from typing import Iterator
@@ -4139,7 +4140,7 @@ def timeout(
     return decorate
 
 
-class _Timeout:
+class _Timeout(Generic[_TimeoutFuncRetType]):
     def __init__(
         self,
         function: Callable[..., _TimeoutFuncRetType],

{paasta_tools-1.30.7.data → paasta_tools-1.30.9.data}/scripts/check_orphans.py

@@ -56,7 +56,7 @@ def get_zk_data(ignored_services: Set[str]) -> SmartstackData:
     zk = KazooClient(hosts=zk_hosts)
     zk.start()
 
-    logger.debug(f"pulling smartstack data from zookeeper")
+    logger.debug("pulling smartstack data from zookeeper")
     zk_data = {}
     services = zk.get_children(PREFIX)
     for service in services:

{paasta_tools-1.30.7.data → paasta_tools-1.30.9.data}/scripts/check_spark_jobs.py

@@ -124,7 +124,7 @@ def format_framework(info):
 def format_message_for_service(service, frameworks):
     output = f"Found the following long-running Spark frameworks associated with service {service}.\n"
     output += (
-        f"Please check why they are still running and terminate if appropriate.\n\n"
+        "Please check why they are still running and terminate if appropriate.\n\n"
     )
     output += "\n".join(format_framework(f) for f in frameworks)
     return output

{paasta_tools-1.30.7.data → paasta_tools-1.30.9.data}/scripts/get_running_task_allocation.py

@@ -1,5 +1,6 @@
 #!python
 import argparse
+import json
 import time
 from typing import Any
 from typing import Dict
@@ -12,7 +13,6 @@ from typing import Optional
 from typing import Tuple
 
 import a_sync
-import simplejson as json
 from kubernetes.client import V1Pod
 from kubernetes.client import V1ResourceRequirements
 

{paasta_tools-1.30.7.data → paasta_tools-1.30.9.data}/scripts/setup_kubernetes_job.py

@@ -52,6 +52,7 @@ from paasta_tools.kubernetes_tools import load_kubernetes_service_config_no_cach
 from paasta_tools.metrics import metrics_lib
 from paasta_tools.utils import decompose_job_id
 from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import DeploymentVersion
 from paasta_tools.utils import InvalidJobNameError
 from paasta_tools.utils import load_system_paasta_config
 from paasta_tools.utils import NoConfigurationForServiceError
@@ -241,7 +242,7 @@ def get_kubernetes_deployment_config(
             service_instance_configs_list.append((True, None))
         except NoConfigurationForServiceError:
             error_msg = (
-                f"Could not read kubernetes configuration file for %s.%s in cluster %s"
+                "Could not read kubernetes configuration file for %s.%s in cluster %s"
                 % (service_instance[0], service_instance[1], cluster)
             )
             log.error(error_msg)
@@ -363,6 +364,14 @@ def setup_kube_deployments(
         for deployment in existing_kube_deployments
     }
 
+    existing_deployment_versions: Dict[
+        Tuple[str, str, str], List[DeploymentVersion]
+    ] = {}
+    for deployment in existing_kube_deployments:
+        existing_deployment_versions.setdefault(
+            (deployment.service, deployment.instance, deployment.namespace), []
+        ).append(deployment.deployment_version)
+
     hpa_overrides = hpa_overrides or {}
 
     applications = [
@@ -379,6 +388,39 @@ def setup_kube_deployments(
         else (_, None)
         for _, service_instance in service_instance_configs_list
     ]
+
+    def sort_key(ok_app: Tuple[bool, Optional[Application]]) -> int:
+        """This will return 1 if the desired deployment_version matches an existing deployment_version, 2 if the deployment is unhealthy and 0
+        otherwise. This will cause applications that need a new deployment_version to be handled first.
+        This prioritizes "real" bounces (developers pushing new service versions) over things that are likely to be big
+        bounces (config-only changes).
+        Most of the time, this won't matter much, as we should get through our backlog quickly, but when there is a
+        backlog, we want to avoid blocking developers.
+        """
+        _, app = ok_app
+        if app:
+            if (
+                app.kube_deployment.deployment_version
+                in existing_deployment_versions.get(
+                    (
+                        app.kube_deployment.service,
+                        app.kube_deployment.instance,
+                        app.kube_deployment.namespace,
+                    ),
+                    [],
+                )
+            ):
+                # Desired version exists, so this is just a configuration update; handle this second.
+                return 1
+            else:
+                # Desired version doesn't exist, handle this first.
+                return 0
+        else:
+            # Handle broken app last.
+            return 2
+
+    applications.sort(key=sort_key)
+
     api_updates = 0
     for _, app in applications:
         if app:

{paasta_tools-1.30.7.dist-info → paasta_tools-1.30.9.dist-info}/METADATA

@@ -1,11 +1,11 @@
 Metadata-Version: 2.4
 Name: paasta-tools
-Version: 1.30.7
+Version: 1.30.9
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com
 Provides: paasta_tools
-Requires-Python: >=3.8.0
+Requires-Python: >=3.9.0
 License-File: LICENSE
 Requires-Dist: a-sync>=0.5.0
 Requires-Dist: aiohttp>=3.5.4