paasta-tools 1.27.0__py3-none-any.whl → 1.35.8__py3-none-any.whl

paasta_tools/kubernetes/remote_run.py

@@ -20,6 +20,7 @@ from typing import Sequence
 from typing import TypedDict
 
 from kubernetes.client import AuthenticationV1TokenRequest
+from kubernetes.client import RbacV1Subject
 from kubernetes.client import V1Job
 from kubernetes.client import V1ObjectMeta
 from kubernetes.client import V1Pod
@@ -28,7 +29,6 @@ from kubernetes.client import V1Role
 from kubernetes.client import V1RoleBinding
 from kubernetes.client import V1RoleRef
 from kubernetes.client import V1ServiceAccount
-from kubernetes.client import V1Subject
 from kubernetes.client import V1TokenRequestSpec
 from kubernetes.client.exceptions import ApiException
 
@@ -44,7 +44,7 @@ from paasta_tools.kubernetes_tools import KubeClient
 from paasta_tools.kubernetes_tools import limit_size_with_hash
 from paasta_tools.kubernetes_tools import paasta_prefixed
 from paasta_tools.utils import load_system_paasta_config
-
+from paasta_tools.utils import NoConfigurationForServiceError
 
 logger = logging.getLogger(__name__)
 REMOTE_RUN_JOB_LABEL = "remote-run"
@@ -66,8 +66,8 @@ class RemoteRunOutcome(TypedDict, total=False):
     namespace: str
 
 
-def _format_remote_run_job_name(
-    job: V1Job,
+def format_remote_run_job_name(
+    job_name: str,
     user: str,
 ) -> str:
     """Format name for remote run job
@@ -76,7 +76,35 @@ def _format_remote_run_job_name(
     :param str user: the user requesting the remote-run
     :return: job name
     """
-    return limit_size_with_hash(f"remote-run-{user}-{job.metadata.name}")
+    return limit_size_with_hash(f"remote-run-{user}-{job_name}")
+
+
+def load_eks_or_adhoc_deployment_config(
+    service: str,
+    instance: str,
+    cluster: str,
+    is_toolbox: bool = False,
+    user: Optional[str] = None,
+) -> EksDeploymentConfig:
+    assert user or not is_toolbox, "User required for toolbox deployment"
+    try:
+        deployment_config = (
+            generate_toolbox_deployment(service, cluster, user)
+            if is_toolbox
+            else load_eks_service_config(service, instance, cluster)
+        )
+    except NoConfigurationForServiceError:
+        # Perhaps they are trying to use an adhoc instance
+        deployment_config = load_adhoc_job_config(service, instance, cluster)
+        deployment_config = EksDeploymentConfig(
+            service,
+            cluster,
+            instance,
+            config_dict=deployment_config.config_dict,
+            branch_dict=deployment_config.branch_dict,
+        )
+        deployment_config.config_filename_prefix = "adhoc"
+    return deployment_config
 
 
 def remote_run_start(
@@ -88,6 +116,7 @@ def remote_run_start(
     recreate: bool,
     max_duration: int,
     is_toolbox: bool,
+    command: Optional[str] = None,
 ) -> RemoteRunOutcome:
     """Trigger remote-run job
 
@@ -99,19 +128,20 @@ def remote_run_start(
     :param bool recreate: whether to recreate remote-run job if existing
     :param int max_duration: maximum allowed duration for the remote-ruh job
     :param bool is_toolbox: requested job is for a toolbox container
+    :param str command: command override to execute in the job container
     :return: outcome of the operation, and resulting Kubernetes pod information
     """
     kube_client = KubeClient()
 
     # Load the service deployment settings
-    deployment_config = (
-        generate_toolbox_deployment(service, cluster, user)
-        if is_toolbox
-        else load_eks_service_config(service, instance, cluster)
+    deployment_config = load_eks_or_adhoc_deployment_config(
+        service, instance, cluster, is_toolbox, user
     )
 
-    # Set to interactive mode
-    if interactive and not is_toolbox:
+    # Set override command, or sleep for interactive mode
+    if command and not is_toolbox:
+        deployment_config.config_dict["cmd"] = command
+    elif interactive and not is_toolbox:
         deployment_config.config_dict["cmd"] = f"sleep {max_duration}"
 
     # Create the app with a new name
@@ -120,10 +150,11 @@ def remote_run_start(
         deadline_seconds=max_duration,
         keep_routable_ip=is_toolbox,
     )
-    job_name = _format_remote_run_job_name(formatted_job, user)
+    job_name = format_remote_run_job_name(formatted_job.metadata.name, user)
     formatted_job.metadata.name = job_name
     app_wrapper = get_application_wrapper(formatted_job)
     app_wrapper.soa_config = deployment_config
+    app_wrapper.ensure_service_account(kube_client)
 
     # Launch pod
     logger.info(f"Starting {job_name}")
@@ -178,10 +209,8 @@ def remote_run_ready(
     kube_client = KubeClient()
 
     # Load the service deployment settings
-    deployment_config = (
-        generate_toolbox_deployment(service, cluster, user)
-        if is_toolbox
-        else load_eks_service_config(service, instance, cluster)
+    deployment_config = load_eks_or_adhoc_deployment_config(
+        service, instance, cluster, is_toolbox, user
     )
     namespace = deployment_config.get_namespace()
 
@@ -225,17 +254,15 @@ def remote_run_stop(
     kube_client = KubeClient()
 
     # Load the service deployment settings
-    deployment_config = (
-        generate_toolbox_deployment(service, cluster, user)
-        if is_toolbox
-        else load_eks_service_config(service, instance, cluster)
+    deployment_config = load_eks_or_adhoc_deployment_config(
+        service, instance, cluster, is_toolbox, user
     )
 
     # Rebuild the job metadata
     formatted_job = deployment_config.format_kubernetes_job(
         job_label=REMOTE_RUN_JOB_LABEL
     )
-    job_name = _format_remote_run_job_name(formatted_job, user)
+    job_name = format_remote_run_job_name(formatted_job.metadata.name, user)
     formatted_job.metadata.name = job_name
 
     # Stop the job
@@ -263,14 +290,14 @@ def remote_run_token(
     kube_client = KubeClient()
 
     # Load the service deployment settings
-    deployment_config = load_eks_service_config(service, instance, cluster)
+    deployment_config = load_eks_or_adhoc_deployment_config(service, instance, cluster)
     namespace = deployment_config.get_namespace()
 
     # Rebuild the job metadata
     formatted_job = deployment_config.format_kubernetes_job(
         job_label=REMOTE_RUN_JOB_LABEL
     )
-    job_name = _format_remote_run_job_name(formatted_job, user)
+    job_name = format_remote_run_job_name(formatted_job.metadata.name, user)
 
     # Find pod and create exec token for it
     pod = find_job_pod(kube_client, namespace, job_name)
@@ -449,7 +476,7 @@ def create_pod_scoped_role(
     role_name = f"remote-run-role-{pod_name_hash}"
     policy = V1PolicyRule(
         verbs=["create", "get"],
-        resources=["pods", "pods/exec"],
+        resources=["pods", "pods/exec", "pods/log"],
         resource_names=[pod_name],
         api_groups=[""],
     )
@@ -495,7 +522,7 @@ def bind_role_to_service_account(
             name=role,
         ),
         subjects=[
-            V1Subject(
+            RbacV1Subject(
                 kind="ServiceAccount",
                 name=service_account,
             ),

paasta_tools/kubernetes_tools.py

@@ -50,6 +50,7 @@ from kubernetes import client as kube_client
 from kubernetes import config as kube_config
 from kubernetes.client import CoreV1Event
 from kubernetes.client import models
+from kubernetes.client import RbacV1Subject
 from kubernetes.client import V1Affinity
 from kubernetes.client import V1AWSElasticBlockStoreVolumeSource
 from kubernetes.client import V1Capabilities
@@ -113,7 +114,6 @@ from kubernetes.client import V1ServiceAccount
 from kubernetes.client import V1ServiceAccountTokenProjection
 from kubernetes.client import V1StatefulSet
 from kubernetes.client import V1StatefulSetSpec
-from kubernetes.client import V1Subject
 from kubernetes.client import V1TCPSocketAction
 from kubernetes.client import V1TopologySpreadConstraint
 from kubernetes.client import V1Volume
@@ -151,6 +151,7 @@ from paasta_tools.long_running_service_tools import METRICS_PROVIDER_PISCINA
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_PROMQL
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_UWSGI
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_UWSGI_V2
+from paasta_tools.long_running_service_tools import METRICS_PROVIDER_WORKER_LOAD
 from paasta_tools.long_running_service_tools import ServiceNamespaceConfig
 from paasta_tools.secret_tools import get_secret_name_from_ref
 from paasta_tools.secret_tools import is_secret_ref
@@ -195,10 +196,8 @@ KUBE_DEPLOY_STATEGY_MAP = {
     "brutal": "RollingUpdate",
 }
 HACHECK_POD_NAME = "hacheck"
-GUNICORN_EXPORTER_POD_NAME = "gunicorn--exporter"
 SIDECAR_CONTAINER_NAMES = [
     HACHECK_POD_NAME,
-    GUNICORN_EXPORTER_POD_NAME,
 ]
 KUBERNETES_NAMESPACE = "paasta"
 PAASTA_WORKLOAD_OWNER = "compute_infra_platform_experience"
@@ -270,6 +269,10 @@ class KubeDeployment(NamedTuple):
     namespace: str
     replicas: Optional[int]
 
+    @property
+    def deployment_version(self) -> DeploymentVersion:
+        return DeploymentVersion(self.git_sha, self.image_version)
+
 
 class KubeCustomResource(NamedTuple):
     service: str
@@ -872,7 +875,10 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
                     ),
                 ),
             )
-        elif provider["type"] == METRICS_PROVIDER_UWSGI_V2:
+        elif provider["type"] in {
+            METRICS_PROVIDER_UWSGI_V2,
+            METRICS_PROVIDER_WORKER_LOAD,
+        }:
             return V2MetricSpec(
                 type="Object",
                 object=V2ObjectMetricSource(
@@ -1068,15 +1074,10 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
             service_namespace_config,
             hacheck_sidecar_volumes,
         )
-        gunicorn_exporter_container = self.get_gunicorn_exporter_sidecar_container(
-            system_paasta_config
-        )
 
         sidecars = []
         if hacheck_container:
             sidecars.append(hacheck_container)
-        if gunicorn_exporter_container:
-            sidecars.append(gunicorn_exporter_container)
         return sidecars
 
     def get_readiness_check_prefix(
@@ -1164,37 +1165,6 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
             )
         return None
 
-    def get_gunicorn_exporter_sidecar_container(
-        self,
-        system_paasta_config: SystemPaastaConfig,
-    ) -> Optional[V1Container]:
-
-        if self.should_use_metrics_provider(METRICS_PROVIDER_GUNICORN):
-            return V1Container(
-                image=system_paasta_config.get_gunicorn_exporter_sidecar_image_url(),
-                resources=self.get_sidecar_resource_requirements(
-                    "gunicorn_exporter", system_paasta_config
-                ),
-                name=GUNICORN_EXPORTER_POD_NAME,
-                env=self.get_kubernetes_environment(),
-                ports=[V1ContainerPort(container_port=9117)],
-                lifecycle=V1Lifecycle(
-                    pre_stop=V1LifecycleHandler(
-                        _exec=V1ExecAction(
-                            command=[
-                                "/bin/sh",
-                                "-c",
-                                # we sleep for the same amount of time as we do after an hadown to ensure that we have accurate
-                                # metrics up until our Pod dies
-                                f"sleep {self.get_hacheck_prestop_sleep_seconds()}",
-                            ]
-                        )
-                    )
-                ),
-            )
-
-        return None
-
     def get_env(
         self, system_paasta_config: Optional["SystemPaastaConfig"] = None
     ) -> Dict[str, str]:
@@ -1460,6 +1430,8 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
         secret_volumes: Sequence[SecretVolume],
         service_namespace_config: ServiceNamespaceConfig,
         include_sidecars: bool = True,
+        include_liveness_probe: bool = True,
+        include_readiness_probe: bool = True,
     ) -> Sequence[V1Container]:
         ports = [self.get_container_port()]
         # MONK-1130
@@ -1485,8 +1457,16 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
                 )
             ),
             name=self.get_sanitised_instance_name(),
-            liveness_probe=self.get_liveness_probe(service_namespace_config),
-            readiness_probe=self.get_readiness_probe(service_namespace_config),
+            liveness_probe=(
+                self.get_liveness_probe(service_namespace_config)
+                if include_liveness_probe
+                else None
+            ),
+            readiness_probe=(
+                self.get_readiness_probe(service_namespace_config)
+                if include_readiness_probe
+                else None
+            ),
             ports=[V1ContainerPort(container_port=port) for port in ports],
             security_context=self.get_security_context(),
             volume_mounts=self.get_volume_mounts(
@@ -1532,7 +1512,7 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
         and the service will be removed from smartstack, which is the same effect we get after running hadown.
         """
 
-        # Everywhere this value is currently used (hacheck sidecar or gunicorn sidecar), we can pretty safely
+        # Everywhere this value is currently used (hacheck sidecar), we can pretty safely
         # assume that the service is in smartstack.
         return self.get_prestop_sleep_seconds(is_in_smartstack=True) + 1
 
@@ -1912,7 +1892,7 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
             if self.get_datastore_credentials_secret_hash():
                 volume_mounts.append(
                     V1VolumeMount(
-                        mount_path=f"/datastore",
+                        mount_path="/datastore",
                         name=self.get_datastore_secret_volume_name(),
                         read_only=True,
                     )
@@ -2165,6 +2145,8 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
                 restart_on_failure=False,
                 include_sidecars=include_sidecars,
                 force_no_routable_ip=not keep_routable_ip,
+                include_liveness_probe=False,
+                include_readiness_probe=False,
             )
             pod_template.metadata.labels.update(additional_labels)
             complete_config = V1Job(
@@ -2290,6 +2272,7 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
             or self.get_prometheus_port() is not None
             or self.should_use_metrics_provider(METRICS_PROVIDER_UWSGI)
             or self.should_use_metrics_provider(METRICS_PROVIDER_GUNICORN)
+            or self.should_use_metrics_provider(METRICS_PROVIDER_WORKER_LOAD)
         ):
             return "true"
         return "false"
@@ -2304,6 +2287,8 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
         restart_on_failure: bool = True,
         include_sidecars: bool = True,
         force_no_routable_ip: bool = False,
+        include_liveness_probe: bool = True,
+        include_readiness_probe: bool = True,
     ) -> V1PodTemplateSpec:
         service_namespace_config = load_service_namespace_config(
             service=self.service, namespace=self.get_nerve_namespace()
@@ -2341,6 +2326,8 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
                 system_paasta_config=system_paasta_config,
                 service_namespace_config=service_namespace_config,
                 include_sidecars=include_sidecars,
+                include_liveness_probe=include_liveness_probe,
+                include_readiness_probe=include_readiness_probe,
             ),
             share_process_namespace=True,
             node_selector=self.get_node_selector(),
@@ -2438,6 +2425,10 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
             "paasta.yelp.com/cluster": self.cluster,
             "yelp.com/owner": "compute_infra_platform_experience",
             "paasta.yelp.com/managed": "true",
+            # NOTE: this is mostly here for autoscaling purposes: we use information from the deploy group
+            # during Prometheus relabeling - but it's not a bad label to have around in general, thus its
+            # inclusion here
+            "paasta.yelp.com/deploy_group": self.get_deploy_group(),
         }
         if service_namespace_config.is_in_smartstack():
             labels["paasta.yelp.com/weight"] = str(self.get_weight())
@@ -2463,22 +2454,13 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
 
         # not all services use autoscaling, so we label those that do in order to have
         # prometheus selectively discover/scrape them
-        if self.should_use_metrics_provider(METRICS_PROVIDER_UWSGI):
-            # UWSGI no longer needs a label to indicate it needs to be scraped as all pods are checked for the uwsgi stats port by our centralized uwsgi-exporter
-            # But we do still need deploy_group for relabeling properly
-            # this should probably eventually be made into a default label,
-            # but for now we're fine with it being behind these feature toggles.
-            # ideally, we'd also have the docker image here for ease-of-use
-            # in Prometheus relabeling, but that information is over the
-            # character limit for k8s labels (63 chars)
-            labels["paasta.yelp.com/deploy_group"] = self.get_deploy_group()
-
-        elif self.should_use_metrics_provider(METRICS_PROVIDER_PISCINA):
-            labels["paasta.yelp.com/deploy_group"] = self.get_deploy_group()
+        # NOTE: these are not mutually exclusive as a service could use multiple autoscaling types
+        if self.should_use_metrics_provider(METRICS_PROVIDER_PISCINA):
             labels["paasta.yelp.com/scrape_piscina_prometheus"] = "true"
 
-        elif self.should_use_metrics_provider(METRICS_PROVIDER_GUNICORN):
-            labels["paasta.yelp.com/deploy_group"] = self.get_deploy_group()
+        if self.should_use_metrics_provider(
+            METRICS_PROVIDER_GUNICORN
+        ) or self.should_use_metrics_provider(METRICS_PROVIDER_WORKER_LOAD):
             labels["paasta.yelp.com/scrape_gunicorn_prometheus"] = "true"
 
         # the default AWS LB Controller behavior is to enable this by-namespace
@@ -3010,7 +2992,7 @@ def ensure_paasta_api_rolebinding(kube_client: KubeClient, namespace: str) -> No
                 name="paasta-api-server-per-namespace",
             ),
             subjects=[
-                V1Subject(
+                RbacV1Subject(
                     kind="User",
                     name="yelp.com/paasta-api-server",
                 ),
@@ -3392,21 +3374,26 @@ def pod_disruption_budget_for_service_instance(
     instance: str,
     max_unavailable: Union[str, int],
     namespace: str,
+    unhealthy_pod_eviction_policy: str,
 ) -> V1PodDisruptionBudget:
+    selector = V1LabelSelector(
+        match_labels={
+            "paasta.yelp.com/service": service,
+            "paasta.yelp.com/instance": instance,
+        }
+    )
+    spec = V1PodDisruptionBudgetSpec(
+        max_unavailable=max_unavailable,
+        unhealthy_pod_eviction_policy=unhealthy_pod_eviction_policy,
+        selector=selector,
+    )
+
     return V1PodDisruptionBudget(
         metadata=V1ObjectMeta(
             name=get_kubernetes_app_name(service, instance),
             namespace=namespace,
         ),
-        spec=V1PodDisruptionBudgetSpec(
-            max_unavailable=max_unavailable,
-            selector=V1LabelSelector(
-                match_labels={
-                    "paasta.yelp.com/service": service,
-                    "paasta.yelp.com/instance": instance,
-                }
-            ),
-        ),
+        spec=spec,
     )
 
 
@@ -4190,6 +4177,10 @@ def create_pod_topology_spread_constraints(
                 when_unsatisfiable=constraint.get(
                     "when_unsatisfiable", "ScheduleAnyway"
                 ),
+                # we might want to default this to someting else in the future
+                # but for now, make this opt-in
+                # (null or empty list means only match against the labelSelector)
+                match_label_keys=constraint.get("match_label_keys", None),
             )
         )
 
@@ -4393,7 +4384,7 @@ def ensure_service_account(
                     name=k8s_role,
                 ),
                 subjects=[
-                    V1Subject(
+                    RbacV1Subject(
                         kind="ServiceAccount",
                         namespace=namespace,
                         name=sa_name,

paasta_tools/long_running_service_tools.py

@@ -41,6 +41,7 @@ DEFAULT_ACTIVE_REQUESTS_AUTOSCALING_MOVING_AVERAGE_WINDOW = 1800
 DEFAULT_UWSGI_AUTOSCALING_MOVING_AVERAGE_WINDOW = 1800
 DEFAULT_PISCINA_AUTOSCALING_MOVING_AVERAGE_WINDOW = 1800
 DEFAULT_GUNICORN_AUTOSCALING_MOVING_AVERAGE_WINDOW = 1800
+DEFAULT_WORKER_LOAD_AUTOSCALING_MOVING_AVERAGE_WINDOW = 1800
 
 METRICS_PROVIDER_CPU = "cpu"
 METRICS_PROVIDER_UWSGI = "uwsgi"
@@ -48,7 +49,8 @@ METRICS_PROVIDER_UWSGI_V2 = "uwsgi-v2"
 METRICS_PROVIDER_GUNICORN = "gunicorn"
 METRICS_PROVIDER_PISCINA = "piscina"
 METRICS_PROVIDER_ACTIVE_REQUESTS = "active-requests"
-METRICS_PROVIDER_PROMQL = "arbitrary_promql"
+METRICS_PROVIDER_PROMQL = "arbitrary-promql"
+METRICS_PROVIDER_WORKER_LOAD = "worker-load"
 
 ALL_METRICS_PROVIDERS = [
     METRICS_PROVIDER_CPU,
@@ -58,6 +60,7 @@ ALL_METRICS_PROVIDERS = [
     METRICS_PROVIDER_PISCINA,
     METRICS_PROVIDER_ACTIVE_REQUESTS,
     METRICS_PROVIDER_PROMQL,
+    METRICS_PROVIDER_WORKER_LOAD,
 ]
 
 
@@ -85,6 +88,7 @@ class LongRunningServiceConfigDict(InstanceConfigDict, total=False):
     bounce_margin_factor: float
     should_ping_for_unhealthy_pods: bool
     weight: int
+    unhealthy_pod_eviction_policy: str
 
 
 class ServiceNamespaceConfig(dict):
@@ -95,9 +99,12 @@ class ServiceNamespaceConfig(dict):
         """
         healthcheck_mode = self.get("healthcheck_mode", None)
         if not healthcheck_mode:
-            return self.get_mode()
-        else:
-            return healthcheck_mode
+            mode = self.get_mode()
+            if mode == "http2":
+                healthcheck_mode = "http"
+            else:
+                healthcheck_mode = mode
+        return healthcheck_mode
 
     def get_mode(self) -> str:
         """Get the mode that the service runs in and check that we support it.
@@ -112,7 +119,7 @@ class ServiceNamespaceConfig(dict):
                 return None
             else:
                 return "http"
-        elif mode in ["http", "tcp", "https"]:
+        elif mode in ["http", "http2", "tcp", "https"]:
             return mode
         else:
             raise InvalidSmartstackMode("Unknown mode: %s" % mode)
@@ -407,6 +414,9 @@ class LongRunningServiceConfig(InstanceConfig):
     def get_bounce_margin_factor(self) -> float:
         return self.config_dict.get("bounce_margin_factor", 0.95)
 
+    def get_unhealthy_pod_eviction_policy(self) -> str:
+        return self.config_dict.get("unhealthy_pod_eviction_policy", "IfHealthyBudget")
+
     def get_should_ping_for_unhealthy_pods(self, default: bool) -> bool:
         return self.config_dict.get("should_ping_for_unhealthy_pods", default)
 

paasta_tools/mesos/master.py

@@ -147,7 +147,7 @@ class MesosMaster:
     def _file_resolver(self, cfg):
         return self.resolve(open(cfg[6:], "r+").read().strip())
 
-    @retry(KazooTimeoutError, tries=5, delay=0.5, logger=logger)
+    @retry(KazooTimeoutError, tries=5, delay=0.5, logger=logger)  # type: ignore
     def _zookeeper_resolver(self, cfg):
         hosts, path = cfg[5:].split("/", 1)
         path = "/" + path

paasta_tools/metrics/metastatus_lib.py

@@ -43,7 +43,6 @@ from paasta_tools.kubernetes_tools import paasta_prefixed
 from paasta_tools.kubernetes_tools import PodStatus
 from paasta_tools.mesos.master import MesosMetrics
 from paasta_tools.mesos.master import MesosState
-from paasta_tools.mesos_maintenance import MAINTENANCE_ROLE
 from paasta_tools.mesos_tools import get_all_tasks_from_state
 from paasta_tools.mesos_tools import get_mesos_quorum
 from paasta_tools.mesos_tools import get_number_of_mesos_masters
@@ -97,9 +96,6 @@ def get_mesos_cpu_status(
     total = metrics["master/cpus_total"]
     used = metrics["master/cpus_used"]
 
-    for slave in mesos_state["slaves"]:
-        used += reserved_maintenence_resources(slave["reserved_resources"])["cpus"]
-
     available = total - used
     return total, used, available
 
@@ -135,9 +131,6 @@ def get_mesos_memory_status(
     total = metrics["master/mem_total"]
     used = metrics["master/mem_used"]
 
-    for slave in mesos_state["slaves"]:
-        used += reserved_maintenence_resources(slave["reserved_resources"])["mem"]
-
     available = total - used
 
     return total, used, available
@@ -176,9 +169,6 @@ def get_mesos_disk_status(
     total = metrics["master/disk_total"]
     used = metrics["master/disk_used"]
 
-    for slave in mesos_state["slaves"]:
-        used += reserved_maintenence_resources(slave["reserved_resources"])["disk"]
-
     available = total - used
     return total, used, available
 
@@ -216,9 +206,6 @@ def get_mesos_gpu_status(
     total = metrics["master/gpus_total"]
     used = metrics["master/gpus_used"]
 
-    for slave in mesos_state["slaves"]:
-        used += reserved_maintenence_resources(slave["reserved_resources"])["gpus"]
-
     available = total - used
     return total, used, available
 
@@ -614,7 +601,7 @@ def group_slaves_by_key_func(
     """
     sorted_slaves: Sequence[_GenericNodeT]
     if sort_func is None:
-        sorted_slaves = sorted(slaves, key=key_func)
+        sorted_slaves = sorted(slaves, key=key_func)  # type: ignore # this code is to be deleted
     else:
         sorted_slaves = sort_func(slaves)
 
@@ -647,11 +634,6 @@ def calculate_resource_utilization_for_slaves(
     for task in tasks:
         task_resources = task["resources"]
         resource_free_dict.subtract(Counter(filter_mesos_state_metrics(task_resources)))
-    for slave in slaves:
-        filtered_resources = filter_mesos_state_metrics(
-            reserved_maintenence_resources(slave["reserved_resources"])
-        )
-        resource_free_dict.subtract(Counter(filtered_resources))
     return {
         "free": ResourceInfo(
             cpus=resource_free_dict["cpus"],
@@ -1102,9 +1084,3 @@ def get_table_rows_for_resource_info_dict(
     return attribute_values + format_row_for_resource_utilization_healthchecks(
         healthcheck_utilization_pairs
     )
-
-
-def reserved_maintenence_resources(
-    resources: MesosResources,
-):
-    return resources.get(MAINTENANCE_ROLE, {"cpus": 0, "mem": 0, "disk": 0, "gpus": 0})

paasta_tools/metrics/metrics_lib.py

@@ -8,6 +8,7 @@ from inspect import currentframe
 from types import TracebackType
 from typing import Any
 from typing import Callable
+from typing import cast
 from typing import Dict
 from typing import Optional
 from typing import Type
@@ -110,10 +111,18 @@ class MeteoriteMetrics(BaseMetrics):
             )
 
     def create_timer(self, name: str, **kwargs: Any) -> TimerProtocol:
-        return yelp_meteorite.create_timer(self.base_name + "." + name, **kwargs)
+        # yelp_meteorite returns an EmptyMetric object if the timer is misconfigured
+        # ...but that doesn't have the same interface ;_;
+        return cast(
+            Timer, yelp_meteorite.create_timer(self.base_name + "." + name, **kwargs)
+        )
 
     def create_gauge(self, name: str, **kwargs: Any) -> GaugeProtocol:
-        return yelp_meteorite.create_gauge(self.base_name + "." + name, **kwargs)
+        # yelp_meteorite returns an EmptyMetric object if the gauge is misconfigured
+        # ...but that doesn't have the same interface ;_;
+        return cast(
+            Gauge, yelp_meteorite.create_gauge(self.base_name + "." + name, **kwargs)
+        )
 
     def create_counter(self, name: str, **kwargs: Any) -> CounterProtocol:
         return yelp_meteorite.create_counter(self.base_name + "." + name, **kwargs)
@@ -163,7 +172,7 @@ class Gauge(GaugeProtocol):
         log.debug(f"gauge {self.name} set to {value}")
 
 
-class Counter(GaugeProtocol):
+class Counter(CounterProtocol):
     def __init__(self, name: str) -> None:
         self.name = name
         self.counter = 0

paasta_tools/paastaapi/__init__.py

@@ -7,7 +7,7 @@
 
     No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)  # noqa: E501
 
-    The version of the OpenAPI document: 1.2.0
+    The version of the OpenAPI document: 1.3.0
     Generated by: https://openapi-generator.tech
 """