paasta-tools 1.22.0__tar.gz → 1.23.1__tar.gz

{paasta_tools-1.22.0/paasta_tools.egg-info → paasta_tools-1.23.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: paasta-tools
-Version: 1.22.0
+Version: 1.23.1
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com
@@ -58,7 +58,7 @@ Requires-Dist: requests-cache>=0.4.10
 Requires-Dist: retry
 Requires-Dist: ruamel.yaml
 Requires-Dist: sensu-plugin
-Requires-Dist: service-configuration-lib>=3.3.2
+Requires-Dist: service-configuration-lib>=3.3.3
 Requires-Dist: signalfx
 Requires-Dist: slackclient>=1.2.1
 Requires-Dist: sticht>=1.1.0

{paasta_tools-1.22.0 → paasta_tools-1.23.1}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.22.0"
+__version__ = "1.23.1"

{paasta_tools-1.22.0 → paasta_tools-1.23.1}/paasta_tools/api/api_docs/swagger.json

@@ -1037,6 +1037,9 @@
                     "404": {
                         "description": "Service instance not found"
                     },
+                    "409": {
+                        "description": "A pod was found but is currently being terminated"
+                    },
                     "500": {
                         "description": "Failure"
                     }

{paasta_tools-1.22.0 → paasta_tools-1.23.1}/paasta_tools/cli/cmds/remote_run.py

@@ -14,6 +14,8 @@
 # limitations under the License.
 import argparse
 import shutil
+import subprocess
+import sys
 import time
 from typing import List
 
@@ -32,9 +34,12 @@ from paasta_tools.utils import load_system_paasta_config
 from paasta_tools.utils import SystemPaastaConfig
 
 
-KUBECTL_CMD_TEMPLATE = (
+KUBECTL_EXEC_CMD_TEMPLATE = (
     "{kubectl_wrapper} --token {token} exec -it -n {namespace} {pod} -- /bin/bash"
 )
+KUBECTL_CP_CMD_TEMPLATE = (
+    "{kubectl_wrapper} --token {token} -n {namespace} cp {filename} {pod}:/tmp/"
+)
 
 
 def _list_services_and_toolboxes() -> List[str]:
@@ -55,7 +60,9 @@ def _list_services_and_toolboxes() -> List[str]:
 def paasta_remote_run_start(
     args: argparse.Namespace,
     system_paasta_config: SystemPaastaConfig,
+    recursed: bool = False,
 ) -> int:
+    status_prefix = "\x1b[2K\r"  # Clear line, carriage return
     client = get_paasta_oapi_client_with_auth(
         cluster=get_paasta_oapi_api_clustername(cluster=args.cluster, is_eks=True),
         system_paasta_config=system_paasta_config,
@@ -95,10 +102,18 @@ def paasta_remote_run_start(
         if poll_response.status == 200:
             print("")
             break
-        print(f"\rStatus: {poll_response.message}", end="")
+        print(f"{status_prefix}Status: {poll_response.message}", end="")
+        if poll_response.status == 404:
+            # Probably indicates a pod was terminating. Now that its gone, retry the whole process
+            if not recursed:
+                print("\nPod finished terminating. Rerunning")
+                return paasta_remote_run_start(args, system_paasta_config, True)
+            else:
+                print("\nSomething went wrong. Pod still not found.")
+                return 1
         time.sleep(10)
     else:
-        print("Timed out while waiting for job to start")
+        print(f"{status_prefix}Timed out while waiting for job to start")
         return 1
 
     if not args.interactive and not args.toolbox:
@@ -120,13 +135,28 @@ def paasta_remote_run_start(
         kubectl_wrapper = f"kubectl-eks-{args.cluster}"
         if not shutil.which(kubectl_wrapper):
             kubectl_wrapper = f"kubectl-{args.cluster}"
-        exec_command = KUBECTL_CMD_TEMPLATE.format(
+        exec_command = KUBECTL_EXEC_CMD_TEMPLATE.format(
             kubectl_wrapper=kubectl_wrapper,
             namespace=poll_response.namespace,
             pod=poll_response.pod_name,
             token=token_response.token,
         )
 
+    if args.copy_file:
+        for filename in args.copy_file:
+            cp_command = KUBECTL_CP_CMD_TEMPLATE.format(
+                kubectl_wrapper=kubectl_wrapper,
+                namespace=poll_response.namespace,
+                pod=poll_response.pod_name,
+                filename=filename,
+                token=token_response.token,
+            ).split(" ")
+            call = subprocess.run(cp_command, capture_output=True)
+            if call.returncode != 0:
+                print("Error copying file to remote-run pod: ", file=sys.stderr)
+                print(call.stderr.decode("utf-8"), file=sys.stderr)
+                return 1
+
     run_interactive_cli(exec_command)
     return 0
 
@@ -232,6 +262,12 @@ def add_subparser(subparsers: argparse._SubParsersAction) -> None:
         type=int,
         default=600,
     )
+    start_parser.add_argument(
+        "--copy-file",
+        help="Adds a local file to /tmp inside the pod",
+        type=str,
+        action="append",
+    )
     stop_parser = subparsers.add_parser(
         "stop",
         help="Stop your remote-run job if it exists",

{paasta_tools-1.22.0 → paasta_tools-1.23.1}/paasta_tools/cli/cmds/spark_run.py

@@ -375,6 +375,20 @@ def add_subparser(subparsers):
         default=False,
     )
 
+    list_parser.add_argument(
+        "--jira-ticket",
+        help=(
+            "The top level jira ticket used to track the project that this spark-job is related to. "
+            "eg: --jira-ticket=PROJ-123. "
+            "Must be passed for all adhoc jobs. "
+            "See https://yelpwiki.yelpcorp.com/spaces/AML/pages/402885641. "
+        ),
+        type=str,
+        required=False,
+        dest="jira_ticket",
+        default=None,
+    )
+
     aws_group = list_parser.add_argument_group(
         title="AWS credentials options",
         description="If --aws-credentials-yaml is specified, it overrides all "
@@ -1383,6 +1397,7 @@ def paasta_spark_run(args: argparse.Namespace) -> int:
         use_eks=True,
         k8s_server_address=k8s_server_address,
         service_account_name=service_account_name,
+        jira_ticket=args.jira_ticket,
     )
 
     return configure_and_run_docker_container(

{paasta_tools-1.22.0 → paasta_tools-1.23.1}/paasta_tools/contrib/paasta_update_soa_memcpu.py

@@ -135,6 +135,11 @@ def cwd(path):
         os.chdir(pwd)
 
 
+def _force_str_to_int(value: str) -> int:
+    """Force convert strings into ints - even if they're technically floats."""
+    return int(float(value))
+
+
 def get_report_from_splunk(creds, app, filename, criteria_filter):
     """Expect a table containing at least the following fields:
     criteria (<service> kubernetes-<cluster_name> <instance>)
@@ -165,29 +170,87 @@ def get_report_from_splunk(creds, app, filename, criteria_filter):
     resp_text = [x for x in resp_text if x]
     resp_text = [json.loads(x) for x in resp_text]
     services_to_update = {}
+
     for d in resp_text:
         if "result" not in d:
             raise ValueError(f"Splunk request didn't return any results: {resp_text}")
         criteria = d["result"]["criteria"]
-        serv = {}
-        serv["service"] = criteria.split(" ")[0]
-        serv["cluster"] = criteria.split(" ")[1]
-        serv["instance"] = criteria.split(" ")[2]
-        serv["owner"] = d["result"].get("service_owner", "Unavailable")
-        serv["date"] = d["result"]["_time"].split(" ")[0]
-        serv["money"] = d["result"].get("estimated_monthly_savings", 0)
-        serv["project"] = d["result"].get("project", "Unavailable")
-        serv["cpus"] = d["result"].get("suggested_cpus")
-        serv["old_cpus"] = d["result"].get("current_cpus")
-        serv["mem"] = d["result"].get("suggested_mem")
-        serv["old_mem"] = d["result"].get("current_mem")
-        serv["disk"] = d["result"].get("suggested_disk")
-        serv["old_disk"] = d["result"].get("current_disk")
-        serv["min_instances"] = d["result"].get("suggested_min_instances")
-        serv["max_instances"] = d["result"].get("suggested_max_instances")
-        serv["hacheck_cpus"] = d["result"].get("suggested_hacheck_cpus")
-        serv["cpu_burst_add"] = d["result"].get("suggested_cpu_burst_add")
-        services_to_update[criteria] = serv
+
+        serv = {
+            "cluster": criteria.split(" ")[1],
+            "date": d["result"]["_time"].split(" ")[0],
+            "instance": criteria.split(" ")[2],
+            "money": d["result"].get("estimated_monthly_savings", 0),
+            "owner": d["result"].get("service_owner", "Unavailable"),
+            "project": d["result"].get("project", "Unavailable"),
+            "service": criteria.split(" ")[0],
+            # only mergeable fields below
+            "cpu_burst_add": d["result"].get("suggested_cpu_burst_add"),
+            "cpus": d["result"].get("suggested_cpus"),
+            "disk": d["result"].get("suggested_disk"),
+            "hacheck_cpus": d["result"].get("suggested_hacheck_cpus"),
+            "max_instances": d["result"].get("suggested_max_instances"),
+            "mem": d["result"].get("suggested_mem"),
+            "min_instances": d["result"].get("suggested_min_instances"),
+            "old_cpus": d["result"].get("current_cpus"),
+            "old_disk": d["result"].get("current_disk"),
+            "old_mem": d["result"].get("current_mem"),
+        }
+
+        # the report we get is all strings, so we need to convert them to the right types
+        field_conversions = {
+            "current_cpus": float,
+            "suggested_cpu_burst_add": float,
+            "suggested_cpus": float,
+            "suggested_disk": int,
+            "suggested_hacheck_cpus": float,
+            "suggested_max_instances": int,
+            "suggested_mem": int,
+            "suggested_min_instances": int,
+            # not quite sure why these are floats...they're ints in soaconfigs
+            "current_disk": _force_str_to_int,
+            "current_mem": _force_str_to_int,
+        }
+
+        # merge results if we've already seen rows for this service
+        # NOTE: this is necessary since the Splunk search can return multiple rows
+        # for the same (service, cluster, instance) tuple as the autotune query
+        # treats certain cpu allocation changes as if the tuple was entirely different.
+        # this is ostensibly due to a theory that if you update resource allocation, existing
+        # autotune data is potentially invalidated - but in practice this ends up hampering
+        # autotune for services with highly variable resource allocation - e.g., we have some services
+        # that have their cpu allocation tweaked by +/-.1 cpu pretty frequently, but then min/max autotune
+        # is never updated.
+        if criteria in services_to_update:
+            for key in serv:
+                # we probably don't want to merge any other fields since they're going to be strings :p
+                if key not in field_conversions:
+                    continue
+
+                last_proposed_suggestion = services_to_update[criteria][key]
+                proposed_suggestion = serv[key]
+
+                # if both are non-null, take the max of the two
+                if (
+                    last_proposed_suggestion is not None
+                    and proposed_suggestion is not None
+                ):
+                    services_to_update[criteria][key] = max(
+                        last_proposed_suggestion,
+                        proposed_suggestion,
+                        key=field_conversions[key],
+                    )
+                # otherwise, if only one of these is non-null, use that one
+                elif last_proposed_suggestion is not None:
+                    services_to_update[criteria][key] = last_proposed_suggestion
+                elif proposed_suggestion is not None:
+                    services_to_update[criteria][key] = proposed_suggestion
+                # otherwise, if we didn't enter any of the above branches, we're essentially leaving in place the
+                # existing None
+
+        # otherwise, simply add the service to the final report
+        else:
+            services_to_update[criteria] = serv
 
     return {
         "search": search,

{paasta_tools-1.22.0 → paasta_tools-1.23.1}/paasta_tools/kubernetes/remote_run.py

@@ -189,6 +189,8 @@ def remote_run_ready(
     if not pod:
         return {"status": 404, "message": "No pod found"}
     if pod.status.phase == "Running":
+        if pod.metadata.deletion_timestamp:
+            return {"status": 409, "message": "Pod is terminating"}
         result: RemoteRunOutcome = {
             "status": 200,
             "message": "Pod ready",

{paasta_tools-1.22.0 → paasta_tools-1.23.1}/paasta_tools/kubernetes_tools.py

@@ -4334,21 +4334,39 @@ def ensure_service_account(
     kube_client: KubeClient,
     k8s_role: Optional[str] = None,
 ) -> None:
+    role_annotation = "eks.amazonaws.com/role-arn"
     sa_name = get_service_account_name(iam_role, k8s_role)
 
-    if not any(
-        sa.metadata and sa.metadata.name == sa_name
-        for sa in get_all_service_accounts(kube_client, namespace)
-    ):
+    existing_sa = None
+    for sa in get_all_service_accounts(kube_client, namespace):
+        if sa.metadata and sa.metadata.name == sa_name:
+            existing_sa = sa
+            break
+    else:
         sa = V1ServiceAccount(
             kind="ServiceAccount",
             metadata=V1ObjectMeta(
                 name=sa_name,
                 namespace=namespace,
-                annotations={"eks.amazonaws.com/role-arn": iam_role},
+                annotations={role_annotation: iam_role},
             ),
         )
         kube_client.core.create_namespaced_service_account(namespace=namespace, body=sa)
+    if existing_sa:
+        if (
+            not sa.metadata.annotations
+            or sa.metadata.annotations.get(role_annotation, None) != iam_role
+        ):
+            # NOTE: we don't annotate SAs apart with anything other
+            # than the pod identity role ARN, so this will remove
+            # any annotations that folks may have manually added
+            sa.metadata.annotations = {role_annotation: iam_role}
+            kube_client.core.patch_namespaced_service_account(
+                namespace=namespace, body=sa, name=sa.metadata.name
+            )
+            log.info(
+                f"Updated ServiceAccount {sa.metadata.name} iam_role to {iam_role}"
+            )
 
     # we're expecting that any Role dynamically associated with a Service Account already exists.
     # at Yelp, this means that we have a version-controlled resource for the Role in Puppet.

{paasta_tools-1.22.0 → paasta_tools-1.23.1/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: paasta-tools
-Version: 1.22.0
+Version: 1.23.1
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com
@@ -58,7 +58,7 @@ Requires-Dist: requests-cache>=0.4.10
 Requires-Dist: retry
 Requires-Dist: ruamel.yaml
 Requires-Dist: sensu-plugin
-Requires-Dist: service-configuration-lib>=3.3.2
+Requires-Dist: service-configuration-lib>=3.3.3
 Requires-Dist: signalfx
 Requires-Dist: slackclient>=1.2.1
 Requires-Dist: sticht>=1.1.0

{paasta_tools-1.22.0 → paasta_tools-1.23.1}/paasta_tools.egg-info/requires.txt

@@ -49,7 +49,7 @@ requests-cache>=0.4.10
 retry
 ruamel.yaml
 sensu-plugin
-service-configuration-lib>=3.3.2
+service-configuration-lib>=3.3.3
 signalfx
 slackclient>=1.2.1
 sticht>=1.1.0

{paasta_tools-1.22.0 → paasta_tools-1.23.1}/requirements-minimal.txt

@@ -56,7 +56,7 @@ requests-cache >= 0.4.10
 retry
 ruamel.yaml
 sensu-plugin
-service-configuration-lib >= 3.3.2
+service-configuration-lib >= 3.3.3
 signalfx
 slackclient >= 1.2.1
 sticht >= 1.1.0

{paasta_tools-1.22.0 → paasta_tools-1.23.1}/tests/test_kubernetes_tools.py

@@ -4844,6 +4844,80 @@ def test_ensure_service_account_existing():
         mock_client.rbac.create_namespaced_role_binding.assert_not_called()
 
 
+def test_ensure_service_account_existing_different_role():
+    old_iam_role = "arn:aws:iam::000000000000:role/some_role"
+    new_iam_role = "arn:aws:iam::000000000000:role/some-role"
+    namespace = "test_namespace"
+    k8s_role = "mega-admin"
+    expected_sa_name = "paasta--arn-aws-iam-000000000000-role-some-role--mega-admin"
+    with mock.patch(
+        "paasta_tools.kubernetes_tools.kube_config.load_kube_config", autospec=True
+    ), mock.patch(
+        "paasta_tools.kubernetes_tools.KubeClient",
+        autospec=False,
+    ) as mock_kube_client:
+        mock_client = mock.Mock()
+        mock_client.core = mock.Mock(spec=kube_client.CoreV1Api)
+        mock_client.rbac = mock.Mock(spec=kube_client.RbacAuthorizationV1Api)
+        mock_client.core.list_namespaced_service_account.return_value = mock.Mock(
+            spec=V1ServiceAccountList
+        )
+        mock_client.core.list_namespaced_service_account.return_value.items = [
+            V1ServiceAccount(
+                kind="ServiceAccount",
+                metadata=V1ObjectMeta(
+                    name=expected_sa_name,
+                    namespace=namespace,
+                    annotations={"eks.amazonaws.com/role-arn": old_iam_role},
+                ),
+            )
+        ]
+        mock_client.rbac.list_namespaced_role_binding.return_value = mock.Mock(
+            spec=V1RoleBinding,
+        )
+        mock_client.rbac.list_namespaced_role_binding.return_value.items = [
+            V1RoleBinding(
+                kind="ServiceAccount",
+                metadata=V1ObjectMeta(
+                    name=expected_sa_name,
+                    namespace=namespace,
+                ),
+                role_ref=V1RoleRef(
+                    api_group="rbac.authorization.k8s.io",
+                    kind="Role",
+                    name=k8s_role,
+                ),
+                subjects=[
+                    V1Subject(
+                        kind="ServiceAccount",
+                        namespace=namespace,
+                        name=expected_sa_name,
+                    )
+                ],
+            )
+        ]
+        mock_kube_client.return_value = mock_client
+        ensure_service_account(
+            new_iam_role,
+            namespace=namespace,
+            kube_client=mock_client,
+            k8s_role=k8s_role,
+        )
+        mock_client.core.create_namespaced_service_account.assert_not_called()
+        mock_client.core.patch_namespaced_service_account.assert_called_once_with(
+            namespace=namespace,
+            body=V1ServiceAccount(
+                kind="ServiceAccount",
+                metadata=V1ObjectMeta(
+                    name=expected_sa_name,
+                    namespace=namespace,
+                    annotations={"eks.amazonaws.com/role-arn": new_iam_role},
+                ),
+            ),
+            name=expected_sa_name,
+        )
+
+
 def test_ensure_service_account_existing_create_rb_only():
     iam_role = "arn:aws:iam::000000000000:role/some_role"
     namespace = "test_namespace"