paasta-tools 1.27.0__py3-none-any.whl → 1.35.8__py3-none-any.whl

paasta_tools/paastaapi/model/smartstack_location.py

@@ -5,7 +5,7 @@
 
     No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)  # noqa: E501
 
-    The version of the OpenAPI document: 1.2.0
+    The version of the OpenAPI document: 1.3.0
     Generated by: https://openapi-generator.tech
 """
 

paasta_tools/paastaapi/model/smartstack_status.py

@@ -5,7 +5,7 @@
 
     No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)  # noqa: E501
 
-    The version of the OpenAPI document: 1.2.0
+    The version of the OpenAPI document: 1.3.0
     Generated by: https://openapi-generator.tech
 """
 

paasta_tools/paastaapi/model/task_tail_lines.py

@@ -5,7 +5,7 @@
 
     No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)  # noqa: E501
 
-    The version of the OpenAPI document: 1.2.0
+    The version of the OpenAPI document: 1.3.0
     Generated by: https://openapi-generator.tech
 """
 

paasta_tools/paastaapi/model_utils.py

@@ -5,7 +5,7 @@
 
     No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)  # noqa: E501
 
-    The version of the OpenAPI document: 1.2.0
+    The version of the OpenAPI document: 1.3.0
     Generated by: https://openapi-generator.tech
 """
 

paasta_tools/paastaapi/rest.py

@@ -5,7 +5,7 @@
 
     No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)  # noqa: E501
 
-    The version of the OpenAPI document: 1.2.0
+    The version of the OpenAPI document: 1.3.0
     Generated by: https://openapi-generator.tech
 """
 

paasta_tools/remote_git.py

@@ -72,8 +72,8 @@ def create_remote_refs(git_url, ref_mutator, force=False):
         determine_wants = ref_mutator
     # We know we don't need to push any objects.
 
-    def generate_pack_contents(have, want):
-        return []
+    def generate_pack_contents(have, want, ofs_delta=None):
+        return 0, []
 
     return client.send_pack(path, determine_wants, generate_pack_contents)
 

paasta_tools/run-paasta-api-in-dev-mode.py

@@ -45,8 +45,8 @@ def main():
             api.main("dev-mode")
     else:
         os.execl(
-            ".tox/py38-linux/bin/python",
-            ".tox/py38-linux/bin/python",
+            ".tox/py310-linux/bin/python",
+            ".tox/py310-linux/bin/python",
             "-m",
             "paasta_tools.api.api",
             *["-D", "-c", cluster, str(port)],

paasta_tools/run-paasta-api-playground.py

@@ -39,8 +39,8 @@ def main():
             api.main("dev-mode")
     else:
         os.execl(
-            ".tox/py38-linux/bin/python",
-            ".tox/py38-linux/bin/python",
+            ".tox/py310-linux/bin/python",
+            ".tox/py310-linux/bin/python",
             "-m",
             "paasta_tools.api.api",
             *["-D", "-c", cluster, str(port)],

paasta_tools/setup_kubernetes_job.py

@@ -52,6 +52,7 @@ from paasta_tools.kubernetes_tools import load_kubernetes_service_config_no_cach
 from paasta_tools.metrics import metrics_lib
 from paasta_tools.utils import decompose_job_id
 from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import DeploymentVersion
 from paasta_tools.utils import InvalidJobNameError
 from paasta_tools.utils import load_system_paasta_config
 from paasta_tools.utils import NoConfigurationForServiceError
@@ -241,7 +242,7 @@ def get_kubernetes_deployment_config(
             service_instance_configs_list.append((True, None))
         except NoConfigurationForServiceError:
             error_msg = (
-                f"Could not read kubernetes configuration file for %s.%s in cluster %s"
+                "Could not read kubernetes configuration file for %s.%s in cluster %s"
                 % (service_instance[0], service_instance[1], cluster)
             )
             log.error(error_msg)
@@ -363,6 +364,14 @@ def setup_kube_deployments(
         for deployment in existing_kube_deployments
     }
 
+    existing_deployment_versions: Dict[
+        Tuple[str, str, str], List[DeploymentVersion]
+    ] = {}
+    for deployment in existing_kube_deployments:
+        existing_deployment_versions.setdefault(
+            (deployment.service, deployment.instance, deployment.namespace), []
+        ).append(deployment.deployment_version)
+
     hpa_overrides = hpa_overrides or {}
 
     applications = [
@@ -379,6 +388,39 @@ def setup_kube_deployments(
         else (_, None)
         for _, service_instance in service_instance_configs_list
     ]
+
+    def sort_key(ok_app: Tuple[bool, Optional[Application]]) -> int:
+        """This will return 1 if the desired deployment_version matches an existing deployment_version, 2 if the deployment is unhealthy and 0
+        otherwise. This will cause applications that need a new deployment_version to be handled first.
+        This prioritizes "real" bounces (developers pushing new service versions) over things that are likely to be big
+        bounces (config-only changes).
+        Most of the time, this won't matter much, as we should get through our backlog quickly, but when there is a
+        backlog, we want to avoid blocking developers.
+        """
+        _, app = ok_app
+        if app:
+            if (
+                app.kube_deployment.deployment_version
+                in existing_deployment_versions.get(
+                    (
+                        app.kube_deployment.service,
+                        app.kube_deployment.instance,
+                        app.kube_deployment.namespace,
+                    ),
+                    [],
+                )
+            ):
+                # Desired version exists, so this is just a configuration update; handle this second.
+                return 1
+            else:
+                # Desired version doesn't exist, handle this first.
+                return 0
+        else:
+            # Handle broken app last.
+            return 2
+
+    applications.sort(key=sort_key)
+
     api_updates = 0
     for _, app in applications:
         if app:

paasta_tools/setup_prometheus_adapter_config.py

@@ -53,6 +53,9 @@ from paasta_tools.long_running_service_tools import (
 from paasta_tools.long_running_service_tools import (
     DEFAULT_UWSGI_AUTOSCALING_MOVING_AVERAGE_WINDOW,
 )
+from paasta_tools.long_running_service_tools import (
+    DEFAULT_WORKER_LOAD_AUTOSCALING_MOVING_AVERAGE_WINDOW,
+)
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_ACTIVE_REQUESTS
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_CPU
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_GUNICORN
@@ -60,6 +63,7 @@ from paasta_tools.long_running_service_tools import METRICS_PROVIDER_PISCINA
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_PROMQL
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_UWSGI
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_UWSGI_V2
+from paasta_tools.long_running_service_tools import METRICS_PROVIDER_WORKER_LOAD
 from paasta_tools.paasta_service_config_loader import PaastaServiceConfigLoader
 from paasta_tools.utils import DEFAULT_SOA_DIR
 from paasta_tools.utils import get_services_for_cluster
@@ -214,6 +218,10 @@ def create_instance_scaling_rule(
         return create_instance_uwsgi_v2_scaling_rule(
             service, instance_config, metrics_provider_config, paasta_cluster
         )
+    if metrics_provider_config["type"] == METRICS_PROVIDER_WORKER_LOAD:
+        return create_instance_worker_load_scaling_rule(
+            service, instance_config, metrics_provider_config, paasta_cluster
+        )
     if metrics_provider_config["type"] == METRICS_PROVIDER_PISCINA:
         return create_instance_piscina_scaling_rule(
             service, instance_config, metrics_provider_config, paasta_cluster
@@ -523,6 +531,80 @@ def create_instance_uwsgi_v2_scaling_rule(
     }
 
 
+def create_instance_worker_load_scaling_rule(
+    service: str,
+    instance_config: KubernetesDeploymentConfig,
+    metrics_provider_config: MetricsProviderDict,
+    paasta_cluster: str,
+) -> PrometheusAdapterRule:
+    """
+    Creates a Prometheus adapter rule config for a given service instance using generic worker_busy metric.
+    """
+    instance = instance_config.instance
+    moving_average_window = metrics_provider_config.get(
+        "moving_average_window_seconds",
+        DEFAULT_WORKER_LOAD_AUTOSCALING_MOVING_AVERAGE_WINDOW,
+    )
+    deployment_name = get_kubernetes_app_name(service=service, instance=instance)
+
+    # In order for autoscaling to work safely while a service migrates from one namespace to another, the HPA needs to
+    # make sure that the deployment in the new namespace is scaled up enough to handle _all_ the load.
+    # This is because once the new deployment is 100% healthy, cleanup_kubernetes_job will delete the deployment out of
+    # the old namespace all at once, suddenly putting all the load onto the deployment in the new namespace.
+    # To ensure this, we must NOT filter on namespace in worker_filter_terms (which is used when calculating total_load.
+    # This makes sure that desired_instances includes load from all namespaces.
+    worker_filter_terms = f"paasta_cluster='{paasta_cluster}',paasta_service='{service}',paasta_instance='{instance}'"
+
+    # k8s:deployment:pods_status_ready is a metric created by summing kube_pod_status_ready
+    # over paasta service/instance/cluster. it counts the number of ready pods in a paasta
+    # deployment.
+    ready_pods = f"""
+        (sum(
+            k8s:deployment:pods_status_ready{{{worker_filter_terms}}} >= 0
+            or
+            max_over_time(
+                k8s:deployment:pods_status_ready{{{worker_filter_terms}}}[{DEFAULT_EXTRAPOLATION_TIME}s]
+            )
+        ) by (kube_deployment))
+    """
+    load_per_instance = f"""
+        avg(
+            worker_busy{{{worker_filter_terms}}}
+        ) by (kube_pod, kube_deployment)
+    """
+    missing_instances = f"""
+        clamp_min(
+            {ready_pods} - count({load_per_instance}) by (kube_deployment),
+            0
+        )
+    """
+    total_load = f"""
+    (
+        sum(
+            {load_per_instance}
+        ) by (kube_deployment)
+        +
+        {missing_instances}
+    )
+    """
+    total_load_smoothed = f"""
+        avg_over_time(
+            (
+                {total_load}
+            )[{moving_average_window}s:]
+        )
+    """
+
+    metric_name = f"{deployment_name}-worker-load-prom"
+
+    return {
+        "name": {"as": metric_name},
+        "seriesQuery": f"worker_busy{{{worker_filter_terms}}}",
+        "resources": {"template": "kube_<<.Resource>>"},
+        "metricsQuery": _minify_promql(total_load_smoothed),
+    }
+
+
 def create_instance_piscina_scaling_rule(
     service: str,
     instance_config: KubernetesDeploymentConfig,

paasta_tools/setup_tron_namespace.py

@@ -114,7 +114,7 @@ def ensure_service_accounts(job_configs: List[TronJobConfig]) -> None:
                     )
 
 
-def main():
+def main() -> None:
     args = parse_args()
     log_level = logging.DEBUG if args.verbose else logging.INFO
     logging.basicConfig(level=log_level)
@@ -232,7 +232,7 @@ def main():
             log.debug(f"Updated {updated}")
 
         if updated_namespaces != new_configs.keys():
-            skipped = set(new_configs.keys()) - set(updated_namespaces.keys())
+            skipped = list(set(new_configs.keys()) - set(updated_namespaces.keys()))
             log.debug(f"Skipped {skipped}")
 
     skipped_report = skipped if args.verbose else len(skipped)

paasta_tools/tron_tools.py

@@ -506,6 +506,9 @@ class TronActionConfig(InstanceConfig):
             # XXX: update PAASTA_RESOURCE_* env vars to use the correct value from spark_args and set
             # these to the correct values for the executors as part of the driver commandline
 
+            # our internal Spark configuration service needs this to determine if any special behavior is required
+            env["SPARK_DRIVER_TYPE"] = "tron"
+
         return env
 
     def get_iam_role(self) -> str:
@@ -1289,7 +1292,7 @@ def load_tron_service_config_no_cache(
 def create_complete_master_config(cluster, soa_dir=DEFAULT_SOA_DIR):
     system_paasta_config = load_system_paasta_config()
     tronfig_folder = get_tronfig_folder(soa_dir=soa_dir, cluster=cluster)
-    config = read_yaml_file(os.path.join(tronfig_folder, f"MASTER.yaml"))
+    config = read_yaml_file(os.path.join(tronfig_folder, "MASTER.yaml"))
     master_config = format_master_config(
         config,
         system_paasta_config.get_volumes(),

paasta_tools/utils.py

@@ -53,6 +53,7 @@ from typing import Collection
 from typing import ContextManager
 from typing import Dict
 from typing import FrozenSet
+from typing import Generic
 from typing import IO
 from typing import Iterable
 from typing import Iterator
@@ -181,6 +182,7 @@ CAPS_DROP = [
     "SYS_CHROOT",
     "SETFCAP",
 ]
+DEFAULT_READONLY_DOCKER_REGISTRY_AUTH_FILE = "/nail/etc/docker-registry-ro"
 
 
 class RollbackTypes(Enum):
@@ -1948,6 +1950,7 @@ class TopologySpreadConstraintDict(TypedDict, total=False):
     topology_key: str
     when_unsatisfiable: Literal["ScheduleAnyway", "DoNotSchedule"]
     max_skew: int
+    match_label_keys: List[str]
 
 
 class SystemPaastaConfigDict(TypedDict, total=False):
@@ -2035,7 +2038,6 @@ class SystemPaastaConfigDict(TypedDict, total=False):
     synapse_port: int
     taskproc: Dict
     tron: Dict
-    gunicorn_exporter_sidecar_image_url: str
     vault_cluster_map: Dict
     vault_environment: str
     volumes: List[DockerVolume]
@@ -2066,6 +2068,9 @@ class SystemPaastaConfigDict(TypedDict, total=False):
     enable_tron_tsc: bool
     default_spark_iam_user: str
     default_spark_driver_pool_override: str
+    readonly_docker_registry_auth_file: str
+    private_docker_registries: List[str]
+    unhealthy_pod_eviction_policy: str
 
 
 def load_system_paasta_config(
@@ -2166,7 +2171,7 @@ class SystemPaastaConfig:
 
     def get_default_spark_iam_user(self) -> str:
         return self.config_dict.get(
-            "default_spark_iam_user", "/etc/boto_cfg/mrjob.yaml"
+            "default_spark_iam_user", "/etc/boto_cfg/spark_driver.yaml"
         )
 
     def get_default_spark_driver_pool_override(self) -> str:
@@ -2668,13 +2673,6 @@ class SystemPaastaConfig:
     def default_should_use_uwsgi_exporter(self) -> bool:
         return self.config_dict.get("default_should_use_uwsgi_exporter", False)
 
-    def get_gunicorn_exporter_sidecar_image_url(self) -> str:
-        """Get the docker image URL for the gunicorn_exporter sidecar container"""
-        return self.config_dict.get(
-            "gunicorn_exporter_sidecar_image_url",
-            "docker-paasta.yelpcorp.com:443/gunicorn_exporter-k8s-sidecar:v0.24.0-yelp0",
-        )
-
     def get_mark_for_deployment_max_polling_threads(self) -> int:
         return self.config_dict.get("mark_for_deployment_max_polling_threads", 4)
 
@@ -2847,6 +2845,26 @@ class SystemPaastaConfig:
             # NOTE: this should never happen unless we've gotten bad data
             return None
 
+    def get_readonly_docker_registry_auth_file(self) -> str:
+        """Get the location of the readonly docker registry auth file as an absolute path."""
+        return self.config_dict.get(
+            "readonly_docker_registry_auth_file",
+            DEFAULT_READONLY_DOCKER_REGISTRY_AUTH_FILE,
+        )
+
+    def get_private_docker_registries(self) -> Set[str]:
+        """Get all the internal Docker registries without generally-available RO creds."""
+        return set(self.config_dict.get("private_docker_registries", []))
+
+    def get_unhealthy_pod_eviction_policy(self) -> str:
+        """
+        Get the unhealthy pod eviction policy for the cluster. Posible values:
+        * IfHealthyBudget: unhealthy pods will only be evicted wrt to PodDisruptionBudget
+        * AlwaysAllow: evict unhealthy pods regardless of the PodDisruptionBudget status.
+        Defaults to IfHealthyBudget
+        """
+        return self.config_dict.get("unhealthy_pod_eviction_policy", "IfHealthyBudget")
+
 
 def _run(
     command: Union[str, List[str]],
@@ -3460,7 +3478,7 @@ def get_docker_host() -> str:
 
 
 def get_docker_client() -> APIClient:
-    client_opts = kwargs_from_env(assert_hostname=False)
+    client_opts = kwargs_from_env()
     if "base_url" in client_opts:
         return APIClient(**client_opts)
     else:
@@ -4139,7 +4157,7 @@ def timeout(
     return decorate
 
 
-class _Timeout:
+class _Timeout(Generic[_TimeoutFuncRetType]):
     def __init__(
         self,
         function: Callable[..., _TimeoutFuncRetType],

paasta_tools/yaml_tools.py

@@ -2,7 +2,7 @@ import sys
 
 import yaml
 
-# try and catch both /opt/venvs/paasta-tools and ~/pg/paasta/.tox/py38-linux as if we're being run as an application,
+# try and catch both /opt/venvs/paasta-tools and ~/pg/paasta/.tox/py310-linux as if we're being run as an application,
 # we likely want to fail on a potential slowdown rather than experience a performance regression
 if "paasta" in sys.prefix:
     from yaml import CSafeLoader as Loader

{paasta_tools-1.27.0.data → paasta_tools-1.35.8.data}/scripts/check_orphans.py

@@ -56,7 +56,7 @@ def get_zk_data(ignored_services: Set[str]) -> SmartstackData:
     zk = KazooClient(hosts=zk_hosts)
     zk.start()
 
-    logger.debug(f"pulling smartstack data from zookeeper")
+    logger.debug("pulling smartstack data from zookeeper")
     zk_data = {}
     services = zk.get_children(PREFIX)
     for service in services:

{paasta_tools-1.27.0.data → paasta_tools-1.35.8.data}/scripts/check_spark_jobs.py

@@ -124,7 +124,7 @@ def format_framework(info):
 def format_message_for_service(service, frameworks):
     output = f"Found the following long-running Spark frameworks associated with service {service}.\n"
     output += (
-        f"Please check why they are still running and terminate if appropriate.\n\n"
+        "Please check why they are still running and terminate if appropriate.\n\n"
     )
     output += "\n".join(format_framework(f) for f in frameworks)
     return output

{paasta_tools-1.27.0.data → paasta_tools-1.35.8.data}/scripts/generate_deployments_for_service.py

@@ -101,6 +101,8 @@ def parse_args() -> argparse.Namespace:
         "--service",
         required=True,
         help="Service name to make the deployments.json for",
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     )
     args = parser.parse_args()
     return args

{paasta_tools-1.27.0.data → paasta_tools-1.35.8.data}/scripts/get_running_task_allocation.py

@@ -1,5 +1,6 @@
 #!python
 import argparse
+import json
 import time
 from typing import Any
 from typing import Dict
@@ -12,7 +13,6 @@ from typing import Optional
 from typing import Tuple
 
 import a_sync
-import simplejson as json
 from kubernetes.client import V1Pod
 from kubernetes.client import V1ResourceRequirements
 

{paasta_tools-1.27.0.data → paasta_tools-1.35.8.data}/scripts/ide_helper.py

@@ -53,12 +53,12 @@ def install_vscode_support() -> None:
                 "python": "${workspaceFolder}/.paasta/bin/python",
                 "program": "${workspaceFolder}/.paasta/bin/tox",
                 "subProcess": True,
-                "args": ["-e", "py38-linux,docs,mypy,tests"],
+                "args": ["-e", "py310-linux,docs,mypy,tests"],
             },
             {
                 "name": "paasta cli",
                 "cwd": "${workspaceFolder}",
-                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "python": "${workspaceFolder}/.tox/py310-linux/bin/python",
                 "type": "python",
                 "request": "launch",
                 "module": "paasta_tools.cli.cli",
@@ -66,7 +66,7 @@ def install_vscode_support() -> None:
             {
                 "name": "paasta rollback",
                 "cwd": "${workspaceFolder}",
-                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "python": "${workspaceFolder}/.tox/py310-linux/bin/python",
                 "type": "python",
                 "request": "launch",
                 "module": "paasta_tools.cli.cli",
@@ -83,7 +83,7 @@ def install_vscode_support() -> None:
             {
                 "name": "paasta mark-for-deployment",
                 "cwd": "${workspaceFolder}",
-                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "python": "${workspaceFolder}/.tox/py310-linux/bin/python",
                 "type": "python",
                 "request": "launch",
                 "module": "paasta_tools.cli.cli",
@@ -101,7 +101,7 @@ def install_vscode_support() -> None:
             {
                 "name": "paasta status",
                 "cwd": "${workspaceFolder}",
-                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "python": "${workspaceFolder}/.tox/py310-linux/bin/python",
                 "type": "python",
                 "request": "launch",
                 "module": "paasta_tools.cli.cli",
@@ -118,7 +118,7 @@ def install_vscode_support() -> None:
             {
                 "name": "paasta playground",
                 "cwd": "${workspaceFolder}",
-                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "python": "${workspaceFolder}/.tox/py310-linux/bin/python",
                 "type": "python",
                 "request": "launch",
                 "module": "paasta_tools.cli.cli",
@@ -138,7 +138,7 @@ def install_vscode_support() -> None:
             {
                 "name": "paasta status playground",
                 "cwd": "${workspaceFolder}",
-                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "python": "${workspaceFolder}/.tox/py310-linux/bin/python",
                 "type": "python",
                 "request": "launch",
                 "module": "paasta_tools.cli.cli",
@@ -157,7 +157,7 @@ def install_vscode_support() -> None:
             {
                 "name": "paasta logs",
                 "cwd": "${workspaceFolder}",
-                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "python": "${workspaceFolder}/.tox/py310-linux/bin/python",
                 "type": "python",
                 "request": "launch",
                 "module": "paasta_tools.cli.cli",
@@ -175,7 +175,7 @@ def install_vscode_support() -> None:
                 "name": "paasta validate",
                 # This command has to be ran from inside the service repo in yelpsoa-configs
                 "cwd": "${userHome}/pg/yelpsoa-configs/",
-                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "python": "${workspaceFolder}/.tox/py310-linux/bin/python",
                 "type": "python",
                 "request": "launch",
                 "module": "paasta_tools.cli.cli",
@@ -184,10 +184,10 @@ def install_vscode_support() -> None:
             {
                 # 1) Follow step 1 in "Running the PaaSTA HTTP API Locally" wiki
                 # 2) Run this "paasta API" test to debug paasta API
-                # 3) Run client command, e.g. PAASTA_SYSTEM_CONFIG_DIR=./etc_paasta_for_development/  .tox/py38-linux/bin/python paasta_tools/cli/cli.py status --clusters norcal-devc --service katamari_test_service
+                # 3) Run client command, e.g. PAASTA_SYSTEM_CONFIG_DIR=./etc_paasta_for_development/  .tox/py310-linux/bin/python paasta_tools/cli/cli.py status --clusters norcal-devc --service katamari_test_service
                 "name": "paasta API",
                 "cwd": "${workspaceFolder}",
-                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "python": "${workspaceFolder}/.tox/py310-linux/bin/python",
                 "type": "python",
                 "request": "launch",
                 "module": "paasta_tools.run-paasta-api-in-dev-mode",
@@ -203,7 +203,7 @@ def install_vscode_support() -> None:
             {
                 "name": "paasta API playground",
                 "cwd": "${workspaceFolder}",
-                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "python": "${workspaceFolder}/.tox/py310-linux/bin/python",
                 "type": "python",
                 "request": "launch",
                 "module": "paasta_tools.run-paasta-api-playground",
@@ -221,7 +221,7 @@ def install_vscode_support() -> None:
             {
                 "name": "Run setup k8s job in playground",
                 "cwd": "${workspaceFolder}",
-                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "python": "${workspaceFolder}/.tox/py310-linux/bin/python",
                 "type": "python",
                 "request": "launch",
                 "module": "paasta_tools.setup_kubernetes_job",
@@ -244,7 +244,7 @@ def install_vscode_support() -> None:
             {
                 "name": "Generate deployments.json in playground",
                 "cwd": "${workspaceFolder}",
-                "python": "${workspaceFolder}/.tox/py38-linux/bin/python",
+                "python": "${workspaceFolder}/.tox/py310-linux/bin/python",
                 "type": "python",
                 "request": "launch",
                 "module": "paasta_tools.generate_deployments_for_service",

{paasta_tools-1.27.0.data → paasta_tools-1.35.8.data}/scripts/paasta_update_soa_memcpu.py

@@ -181,6 +181,9 @@ def get_report_from_splunk(creds, app, filename, criteria_filter):
             "date": d["result"]["_time"].split(" ")[0],
             "instance": criteria.split(" ")[2],
             "money": d["result"].get("estimated_monthly_savings", 0),
+            "old_cpus": d["result"].get("current_cpus"),
+            "old_disk": d["result"].get("current_disk"),
+            "old_mem": d["result"].get("current_mem"),
             "owner": d["result"].get("service_owner", "Unavailable"),
             "project": d["result"].get("project", "Unavailable"),
             "service": criteria.split(" ")[0],
@@ -192,24 +195,17 @@ def get_report_from_splunk(creds, app, filename, criteria_filter):
             "max_instances": d["result"].get("suggested_max_instances"),
             "mem": d["result"].get("suggested_mem"),
             "min_instances": d["result"].get("suggested_min_instances"),
-            "old_cpus": d["result"].get("current_cpus"),
-            "old_disk": d["result"].get("current_disk"),
-            "old_mem": d["result"].get("current_mem"),
         }
 
         # the report we get is all strings, so we need to convert them to the right types
         field_conversions = {
-            "current_cpus": float,
-            "suggested_cpu_burst_add": float,
-            "suggested_cpus": float,
-            "suggested_disk": int,
-            "suggested_hacheck_cpus": float,
-            "suggested_max_instances": int,
-            "suggested_mem": int,
-            "suggested_min_instances": int,
-            # not quite sure why these are floats...they're ints in soaconfigs
-            "current_disk": _force_str_to_int,
-            "current_mem": _force_str_to_int,
+            "cpus": float,
+            "cpu_burst_add": float,
+            "disk": int,
+            "hacheck_cpus": float,
+            "max_instances": int,
+            "mem": int,
+            "min_instances": int,
         }
 
         # merge results if we've already seen rows for this service