paasta-tools 1.23.0__tar.gz → 1.23.1__tar.gz

{paasta_tools-1.23.0/paasta_tools.egg-info → paasta_tools-1.23.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: paasta-tools
-Version: 1.23.0
+Version: 1.23.1
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com

{paasta_tools-1.23.0 → paasta_tools-1.23.1}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.23.0"
+__version__ = "1.23.1"

{paasta_tools-1.23.0 → paasta_tools-1.23.1}/paasta_tools/contrib/paasta_update_soa_memcpu.py

@@ -135,6 +135,11 @@ def cwd(path):
         os.chdir(pwd)
 
 
+def _force_str_to_int(value: str) -> int:
+    """Force convert strings into ints - even if they're technically floats."""
+    return int(float(value))
+
+
 def get_report_from_splunk(creds, app, filename, criteria_filter):
     """Expect a table containing at least the following fields:
     criteria (<service> kubernetes-<cluster_name> <instance>)
@@ -165,29 +170,87 @@ def get_report_from_splunk(creds, app, filename, criteria_filter):
     resp_text = [x for x in resp_text if x]
     resp_text = [json.loads(x) for x in resp_text]
     services_to_update = {}
+
     for d in resp_text:
         if "result" not in d:
             raise ValueError(f"Splunk request didn't return any results: {resp_text}")
         criteria = d["result"]["criteria"]
-        serv = {}
-        serv["service"] = criteria.split(" ")[0]
-        serv["cluster"] = criteria.split(" ")[1]
-        serv["instance"] = criteria.split(" ")[2]
-        serv["owner"] = d["result"].get("service_owner", "Unavailable")
-        serv["date"] = d["result"]["_time"].split(" ")[0]
-        serv["money"] = d["result"].get("estimated_monthly_savings", 0)
-        serv["project"] = d["result"].get("project", "Unavailable")
-        serv["cpus"] = d["result"].get("suggested_cpus")
-        serv["old_cpus"] = d["result"].get("current_cpus")
-        serv["mem"] = d["result"].get("suggested_mem")
-        serv["old_mem"] = d["result"].get("current_mem")
-        serv["disk"] = d["result"].get("suggested_disk")
-        serv["old_disk"] = d["result"].get("current_disk")
-        serv["min_instances"] = d["result"].get("suggested_min_instances")
-        serv["max_instances"] = d["result"].get("suggested_max_instances")
-        serv["hacheck_cpus"] = d["result"].get("suggested_hacheck_cpus")
-        serv["cpu_burst_add"] = d["result"].get("suggested_cpu_burst_add")
-        services_to_update[criteria] = serv
+
+        serv = {
+            "cluster": criteria.split(" ")[1],
+            "date": d["result"]["_time"].split(" ")[0],
+            "instance": criteria.split(" ")[2],
+            "money": d["result"].get("estimated_monthly_savings", 0),
+            "owner": d["result"].get("service_owner", "Unavailable"),
+            "project": d["result"].get("project", "Unavailable"),
+            "service": criteria.split(" ")[0],
+            # only mergeable fields below
+            "cpu_burst_add": d["result"].get("suggested_cpu_burst_add"),
+            "cpus": d["result"].get("suggested_cpus"),
+            "disk": d["result"].get("suggested_disk"),
+            "hacheck_cpus": d["result"].get("suggested_hacheck_cpus"),
+            "max_instances": d["result"].get("suggested_max_instances"),
+            "mem": d["result"].get("suggested_mem"),
+            "min_instances": d["result"].get("suggested_min_instances"),
+            "old_cpus": d["result"].get("current_cpus"),
+            "old_disk": d["result"].get("current_disk"),
+            "old_mem": d["result"].get("current_mem"),
+        }
+
+        # the report we get is all strings, so we need to convert them to the right types
+        field_conversions = {
+            "current_cpus": float,
+            "suggested_cpu_burst_add": float,
+            "suggested_cpus": float,
+            "suggested_disk": int,
+            "suggested_hacheck_cpus": float,
+            "suggested_max_instances": int,
+            "suggested_mem": int,
+            "suggested_min_instances": int,
+            # not quite sure why these are floats...they're ints in soaconfigs
+            "current_disk": _force_str_to_int,
+            "current_mem": _force_str_to_int,
+        }
+
+        # merge results if we've already seen rows for this service
+        # NOTE: this is necessary since the Splunk search can return multiple rows
+        # for the same (service, cluster, instance) tuple as the autotune query
+        # treats certain cpu allocation changes as if the tuple was entirely different.
+        # this is ostensibly due to a theory that if you update resource allocation, existing
+        # autotune data is potentially invalidated - but in practice this ends up hampering
+        # autotune for services with highly variable resource allocation - e.g., we have some services
+        # that have their cpu allocation tweaked by +/-.1 cpu pretty frequently, but then min/max autotune
+        # is never updated.
+        if criteria in services_to_update:
+            for key in serv:
+                # we probably don't want to merge any other fields since they're going to be strings :p
+                if key not in field_conversions:
+                    continue
+
+                last_proposed_suggestion = services_to_update[criteria][key]
+                proposed_suggestion = serv[key]
+
+                # if both are non-null, take the max of the two
+                if (
+                    last_proposed_suggestion is not None
+                    and proposed_suggestion is not None
+                ):
+                    services_to_update[criteria][key] = max(
+                        last_proposed_suggestion,
+                        proposed_suggestion,
+                        key=field_conversions[key],
+                    )
+                # otherwise, if only one of these is non-null, use that one
+                elif last_proposed_suggestion is not None:
+                    services_to_update[criteria][key] = last_proposed_suggestion
+                elif proposed_suggestion is not None:
+                    services_to_update[criteria][key] = proposed_suggestion
+                # otherwise, if we didn't enter any of the above branches, we're essentially leaving in place the
+                # existing None
+
+        # otherwise, simply add the service to the final report
+        else:
+            services_to_update[criteria] = serv
 
     return {
         "search": search,

{paasta_tools-1.23.0 → paasta_tools-1.23.1}/paasta_tools/kubernetes_tools.py

@@ -4334,21 +4334,39 @@ def ensure_service_account(
     kube_client: KubeClient,
     k8s_role: Optional[str] = None,
 ) -> None:
+    role_annotation = "eks.amazonaws.com/role-arn"
     sa_name = get_service_account_name(iam_role, k8s_role)
 
-    if not any(
-        sa.metadata and sa.metadata.name == sa_name
-        for sa in get_all_service_accounts(kube_client, namespace)
-    ):
+    existing_sa = None
+    for sa in get_all_service_accounts(kube_client, namespace):
+        if sa.metadata and sa.metadata.name == sa_name:
+            existing_sa = sa
+            break
+    else:
         sa = V1ServiceAccount(
             kind="ServiceAccount",
             metadata=V1ObjectMeta(
                 name=sa_name,
                 namespace=namespace,
-                annotations={"eks.amazonaws.com/role-arn": iam_role},
+                annotations={role_annotation: iam_role},
             ),
         )
         kube_client.core.create_namespaced_service_account(namespace=namespace, body=sa)
+    if existing_sa:
+        if (
+            not sa.metadata.annotations
+            or sa.metadata.annotations.get(role_annotation, None) != iam_role
+        ):
+            # NOTE: we don't annotate SAs apart with anything other
+            # than the pod identity role ARN, so this will remove
+            # any annotations that folks may have manually added
+            sa.metadata.annotations = {role_annotation: iam_role}
+            kube_client.core.patch_namespaced_service_account(
+                namespace=namespace, body=sa, name=sa.metadata.name
+            )
+            log.info(
+                f"Updated ServiceAccount {sa.metadata.name} iam_role to {iam_role}"
+            )
 
     # we're expecting that any Role dynamically associated with a Service Account already exists.
     # at Yelp, this means that we have a version-controlled resource for the Role in Puppet.

{paasta_tools-1.23.0 → paasta_tools-1.23.1/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: paasta-tools
-Version: 1.23.0
+Version: 1.23.1
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com

{paasta_tools-1.23.0 → paasta_tools-1.23.1}/tests/test_kubernetes_tools.py

@@ -4844,6 +4844,80 @@ def test_ensure_service_account_existing():
         mock_client.rbac.create_namespaced_role_binding.assert_not_called()
 
 
+def test_ensure_service_account_existing_different_role():
+    old_iam_role = "arn:aws:iam::000000000000:role/some_role"
+    new_iam_role = "arn:aws:iam::000000000000:role/some-role"
+    namespace = "test_namespace"
+    k8s_role = "mega-admin"
+    expected_sa_name = "paasta--arn-aws-iam-000000000000-role-some-role--mega-admin"
+    with mock.patch(
+        "paasta_tools.kubernetes_tools.kube_config.load_kube_config", autospec=True
+    ), mock.patch(
+        "paasta_tools.kubernetes_tools.KubeClient",
+        autospec=False,
+    ) as mock_kube_client:
+        mock_client = mock.Mock()
+        mock_client.core = mock.Mock(spec=kube_client.CoreV1Api)
+        mock_client.rbac = mock.Mock(spec=kube_client.RbacAuthorizationV1Api)
+        mock_client.core.list_namespaced_service_account.return_value = mock.Mock(
+            spec=V1ServiceAccountList
+        )
+        mock_client.core.list_namespaced_service_account.return_value.items = [
+            V1ServiceAccount(
+                kind="ServiceAccount",
+                metadata=V1ObjectMeta(
+                    name=expected_sa_name,
+                    namespace=namespace,
+                    annotations={"eks.amazonaws.com/role-arn": old_iam_role},
+                ),
+            )
+        ]
+        mock_client.rbac.list_namespaced_role_binding.return_value = mock.Mock(
+            spec=V1RoleBinding,
+        )
+        mock_client.rbac.list_namespaced_role_binding.return_value.items = [
+            V1RoleBinding(
+                kind="ServiceAccount",
+                metadata=V1ObjectMeta(
+                    name=expected_sa_name,
+                    namespace=namespace,
+                ),
+                role_ref=V1RoleRef(
+                    api_group="rbac.authorization.k8s.io",
+                    kind="Role",
+                    name=k8s_role,
+                ),
+                subjects=[
+                    V1Subject(
+                        kind="ServiceAccount",
+                        namespace=namespace,
+                        name=expected_sa_name,
+                    )
+                ],
+            )
+        ]
+        mock_kube_client.return_value = mock_client
+        ensure_service_account(
+            new_iam_role,
+            namespace=namespace,
+            kube_client=mock_client,
+            k8s_role=k8s_role,
+        )
+        mock_client.core.create_namespaced_service_account.assert_not_called()
+        mock_client.core.patch_namespaced_service_account.assert_called_once_with(
+            namespace=namespace,
+            body=V1ServiceAccount(
+                kind="ServiceAccount",
+                metadata=V1ObjectMeta(
+                    name=expected_sa_name,
+                    namespace=namespace,
+                    annotations={"eks.amazonaws.com/role-arn": new_iam_role},
+                ),
+            ),
+            name=expected_sa_name,
+        )
+
+
 def test_ensure_service_account_existing_create_rb_only():
     iam_role = "arn:aws:iam::000000000000:role/some_role"
     namespace = "test_namespace"