paasta-tools 1.23.0__tar.gz → 1.23.2__tar.gz

{paasta_tools-1.23.0/paasta_tools.egg-info → paasta_tools-1.23.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: paasta-tools
-Version: 1.23.0
+Version: 1.23.2
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com

{paasta_tools-1.23.0 → paasta_tools-1.23.2}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.23.0"
+__version__ = "1.23.2"

{paasta_tools-1.23.0 → paasta_tools-1.23.2}/paasta_tools/api/api_docs/swagger.json

@@ -930,7 +930,15 @@
                         }
                     },
                     "403": {
-                        "description": "Unauthorized to remote-run this service"
+                        "description": "Unauthorized to remote-run this service",
+                        "schema": {
+                            "type": "object",
+                            "properties": {
+                                "reason": {
+                                    "type": "string"
+                                }
+                            }
+                        }
                     },
                     "404": {
                         "description": "Deployment key not found"
@@ -981,7 +989,15 @@
                         }
                     },
                     "403": {
-                        "description": "Unauthorized to remote-run this service"
+                        "description": "Unauthorized to remote-run this service",
+                        "schema": {
+                            "type": "object",
+                            "properties": {
+                                "reason": {
+                                    "type": "string"
+                                }
+                            }
+                        }
                     },
                     "404": {
                         "description": "Service instance not found"

{paasta_tools-1.23.0 → paasta_tools-1.23.2}/paasta_tools/cli/cmds/remote_run.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import argparse
+import json
 import shutil
 import subprocess
 import sys
@@ -24,6 +25,7 @@ from paasta_tools.cli.utils import get_paasta_oapi_client_with_auth
 from paasta_tools.cli.utils import lazy_choices_completer
 from paasta_tools.cli.utils import run_interactive_cli
 from paasta_tools.kubernetes.remote_run import TOOLBOX_MOCK_SERVICE
+from paasta_tools.paastaapi.exceptions import ApiException
 from paasta_tools.paastaapi.model.remote_run_start import RemoteRunStart
 from paasta_tools.paastaapi.model.remote_run_stop import RemoteRunStop
 from paasta_tools.utils import get_username
@@ -57,6 +59,18 @@ def _list_services_and_toolboxes() -> List[str]:
     )
 
 
+def parse_error(body: str) -> str:
+    try:
+        body_object = json.loads(body)
+    except json.decoder.JSONDecodeError:
+        return body
+    return (
+        body_object.get("reason")
+        or body_object.get("message")
+        or json.dumps(body_object, indent=4)
+    )
+
+
 def paasta_remote_run_start(
     args: argparse.Namespace,
     system_paasta_config: SystemPaastaConfig,
@@ -72,19 +86,21 @@ def paasta_remote_run_start(
         return 1
 
     user = get_username()
-    start_response = client.remote_run.remote_run_start(
-        args.service,
-        args.instance,
-        RemoteRunStart(
-            user=user,
-            interactive=args.interactive,
-            recreate=args.recreate,
-            max_duration=args.max_duration,
-            toolbox=args.toolbox,
-        ),
-    )
-    if start_response.status >= 300:
-        print(f"Error from PaaSTA APIs while starting job: {start_response.message}")
+    try:
+        start_response = client.remote_run.remote_run_start(
+            args.service,
+            args.instance,
+            RemoteRunStart(
+                user=user,
+                interactive=args.interactive,
+                recreate=args.recreate,
+                max_duration=args.max_duration,
+                toolbox=args.toolbox,
+            ),
+        )
+    except ApiException as e:
+        error_msg = parse_error(e.body)
+        print(f"Error from PaaSTA APIs while starting job: {error_msg}")
         return 1
 
     print(
@@ -172,13 +188,18 @@ def paasta_remote_run_stop(
     if not client:
         print("Cannot get a paasta-api client")
         return 1
-    response = client.remote_run.remote_run_stop(
-        args.service,
-        args.instance,
-        RemoteRunStop(user=get_username(), toolbox=args.toolbox),
-    )
+    try:
+        response = client.remote_run.remote_run_stop(
+            args.service,
+            args.instance,
+            RemoteRunStop(user=get_username(), toolbox=args.toolbox),
+        )
+    except ApiException as e:
+        error_msg = parse_error(e.body)
+        print(f"Error from PaaSTA APIs while stopping job: {error_msg}")
+        return 1
     print(response.message)
-    return 0 if response.status < 300 else 1
+    return 0
 
 
 def add_common_args_to_parser(parser: argparse.ArgumentParser):

{paasta_tools-1.23.0 → paasta_tools-1.23.2}/paasta_tools/contrib/paasta_update_soa_memcpu.py

@@ -135,6 +135,11 @@ def cwd(path):
         os.chdir(pwd)
 
 
+def _force_str_to_int(value: str) -> int:
+    """Force convert strings into ints - even if they're technically floats."""
+    return int(float(value))
+
+
 def get_report_from_splunk(creds, app, filename, criteria_filter):
     """Expect a table containing at least the following fields:
     criteria (<service> kubernetes-<cluster_name> <instance>)
@@ -165,29 +170,87 @@ def get_report_from_splunk(creds, app, filename, criteria_filter):
     resp_text = [x for x in resp_text if x]
     resp_text = [json.loads(x) for x in resp_text]
     services_to_update = {}
+
     for d in resp_text:
         if "result" not in d:
             raise ValueError(f"Splunk request didn't return any results: {resp_text}")
         criteria = d["result"]["criteria"]
-        serv = {}
-        serv["service"] = criteria.split(" ")[0]
-        serv["cluster"] = criteria.split(" ")[1]
-        serv["instance"] = criteria.split(" ")[2]
-        serv["owner"] = d["result"].get("service_owner", "Unavailable")
-        serv["date"] = d["result"]["_time"].split(" ")[0]
-        serv["money"] = d["result"].get("estimated_monthly_savings", 0)
-        serv["project"] = d["result"].get("project", "Unavailable")
-        serv["cpus"] = d["result"].get("suggested_cpus")
-        serv["old_cpus"] = d["result"].get("current_cpus")
-        serv["mem"] = d["result"].get("suggested_mem")
-        serv["old_mem"] = d["result"].get("current_mem")
-        serv["disk"] = d["result"].get("suggested_disk")
-        serv["old_disk"] = d["result"].get("current_disk")
-        serv["min_instances"] = d["result"].get("suggested_min_instances")
-        serv["max_instances"] = d["result"].get("suggested_max_instances")
-        serv["hacheck_cpus"] = d["result"].get("suggested_hacheck_cpus")
-        serv["cpu_burst_add"] = d["result"].get("suggested_cpu_burst_add")
-        services_to_update[criteria] = serv
+
+        serv = {
+            "cluster": criteria.split(" ")[1],
+            "date": d["result"]["_time"].split(" ")[0],
+            "instance": criteria.split(" ")[2],
+            "money": d["result"].get("estimated_monthly_savings", 0),
+            "owner": d["result"].get("service_owner", "Unavailable"),
+            "project": d["result"].get("project", "Unavailable"),
+            "service": criteria.split(" ")[0],
+            # only mergeable fields below
+            "cpu_burst_add": d["result"].get("suggested_cpu_burst_add"),
+            "cpus": d["result"].get("suggested_cpus"),
+            "disk": d["result"].get("suggested_disk"),
+            "hacheck_cpus": d["result"].get("suggested_hacheck_cpus"),
+            "max_instances": d["result"].get("suggested_max_instances"),
+            "mem": d["result"].get("suggested_mem"),
+            "min_instances": d["result"].get("suggested_min_instances"),
+            "old_cpus": d["result"].get("current_cpus"),
+            "old_disk": d["result"].get("current_disk"),
+            "old_mem": d["result"].get("current_mem"),
+        }
+
+        # the report we get is all strings, so we need to convert them to the right types
+        field_conversions = {
+            "current_cpus": float,
+            "suggested_cpu_burst_add": float,
+            "suggested_cpus": float,
+            "suggested_disk": int,
+            "suggested_hacheck_cpus": float,
+            "suggested_max_instances": int,
+            "suggested_mem": int,
+            "suggested_min_instances": int,
+            # not quite sure why these are floats...they're ints in soaconfigs
+            "current_disk": _force_str_to_int,
+            "current_mem": _force_str_to_int,
+        }
+
+        # merge results if we've already seen rows for this service
+        # NOTE: this is necessary since the Splunk search can return multiple rows
+        # for the same (service, cluster, instance) tuple as the autotune query
+        # treats certain cpu allocation changes as if the tuple was entirely different.
+        # this is ostensibly due to a theory that if you update resource allocation, existing
+        # autotune data is potentially invalidated - but in practice this ends up hampering
+        # autotune for services with highly variable resource allocation - e.g., we have some services
+        # that have their cpu allocation tweaked by +/-.1 cpu pretty frequently, but then min/max autotune
+        # is never updated.
+        if criteria in services_to_update:
+            for key in serv:
+                # we probably don't want to merge any other fields since they're going to be strings :p
+                if key not in field_conversions:
+                    continue
+
+                last_proposed_suggestion = services_to_update[criteria][key]
+                proposed_suggestion = serv[key]
+
+                # if both are non-null, take the max of the two
+                if (
+                    last_proposed_suggestion is not None
+                    and proposed_suggestion is not None
+                ):
+                    services_to_update[criteria][key] = max(
+                        last_proposed_suggestion,
+                        proposed_suggestion,
+                        key=field_conversions[key],
+                    )
+                # otherwise, if only one of these is non-null, use that one
+                elif last_proposed_suggestion is not None:
+                    services_to_update[criteria][key] = last_proposed_suggestion
+                elif proposed_suggestion is not None:
+                    services_to_update[criteria][key] = proposed_suggestion
+                # otherwise, if we didn't enter any of the above branches, we're essentially leaving in place the
+                # existing None
+
+        # otherwise, simply add the service to the final report
+        else:
+            services_to_update[criteria] = serv
 
     return {
         "search": search,

{paasta_tools-1.23.0 → paasta_tools-1.23.2}/paasta_tools/kubernetes_tools.py

@@ -4334,21 +4334,39 @@ def ensure_service_account(
     kube_client: KubeClient,
     k8s_role: Optional[str] = None,
 ) -> None:
+    role_annotation = "eks.amazonaws.com/role-arn"
     sa_name = get_service_account_name(iam_role, k8s_role)
 
-    if not any(
-        sa.metadata and sa.metadata.name == sa_name
-        for sa in get_all_service_accounts(kube_client, namespace)
-    ):
+    existing_sa = None
+    for sa in get_all_service_accounts(kube_client, namespace):
+        if sa.metadata and sa.metadata.name == sa_name:
+            existing_sa = sa
+            break
+    else:
         sa = V1ServiceAccount(
             kind="ServiceAccount",
             metadata=V1ObjectMeta(
                 name=sa_name,
                 namespace=namespace,
-                annotations={"eks.amazonaws.com/role-arn": iam_role},
+                annotations={role_annotation: iam_role},
             ),
         )
         kube_client.core.create_namespaced_service_account(namespace=namespace, body=sa)
+    if existing_sa:
+        if (
+            not sa.metadata.annotations
+            or sa.metadata.annotations.get(role_annotation, None) != iam_role
+        ):
+            # NOTE: we don't annotate SAs apart with anything other
+            # than the pod identity role ARN, so this will remove
+            # any annotations that folks may have manually added
+            sa.metadata.annotations = {role_annotation: iam_role}
+            kube_client.core.patch_namespaced_service_account(
+                namespace=namespace, body=sa, name=sa.metadata.name
+            )
+            log.info(
+                f"Updated ServiceAccount {sa.metadata.name} iam_role to {iam_role}"
+            )
 
     # we're expecting that any Role dynamically associated with a Service Account already exists.
     # at Yelp, this means that we have a version-controlled resource for the Role in Puppet.

{paasta_tools-1.23.0 → paasta_tools-1.23.2}/paasta_tools/paastaapi/api/remote_run_api.py

@@ -23,6 +23,7 @@ from paasta_tools.paastaapi.model_utils import (  # noqa: F401
     none_type,
     validate_and_convert_types
 )
+from paasta_tools.paastaapi.model.inline_response403 import InlineResponse403
 from paasta_tools.paastaapi.model.remote_run_outcome import RemoteRunOutcome
 from paasta_tools.paastaapi.model.remote_run_start import RemoteRunStart
 from paasta_tools.paastaapi.model.remote_run_stop import RemoteRunStop

paasta_tools-1.23.2/paasta_tools/paastaapi/model/inline_response403.py

@@ -0,0 +1,170 @@
+# coding: utf-8
+
+"""
+    Paasta API
+
+    No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)  # noqa: E501
+
+    The version of the OpenAPI document: 1.2.0
+    Generated by: https://openapi-generator.tech
+"""
+
+
+import re  # noqa: F401
+import sys  # noqa: F401
+
+import nulltype  # noqa: F401
+
+from paasta_tools.paastaapi.model_utils import (  # noqa: F401
+    ApiTypeError,
+    ModelComposed,
+    ModelNormal,
+    ModelSimple,
+    cached_property,
+    change_keys_js_to_python,
+    convert_js_args_to_python_args,
+    date,
+    datetime,
+    file_type,
+    none_type,
+    validate_get_composed_info,
+)
+
+
+class InlineResponse403(ModelNormal):
+    """NOTE: This class is auto generated by OpenAPI Generator.
+    Ref: https://openapi-generator.tech
+
+    Do not edit the class manually.
+
+    Attributes:
+      allowed_values (dict): The key is the tuple path to the attribute
+          and the for var_name this is (var_name,). The value is a dict
+          with a capitalized key describing the allowed value and an allowed
+          value. These dicts store the allowed enum values.
+      attribute_map (dict): The key is attribute name
+          and the value is json key in definition.
+      discriminator_value_class_map (dict): A dict to go from the discriminator
+          variable value to the discriminator class name.
+      validations (dict): The key is the tuple path to the attribute
+          and the for var_name this is (var_name,). The value is a dict
+          that stores validations for max_length, min_length, max_items,
+          min_items, exclusive_maximum, inclusive_maximum, exclusive_minimum,
+          inclusive_minimum, and regex.
+      additional_properties_type (tuple): A tuple of classes accepted
+          as additional properties values.
+    """
+
+    allowed_values = {
+    }
+
+    validations = {
+    }
+
+    additional_properties_type = None
+
+    _nullable = False
+
+    @cached_property
+    def openapi_types():
+        """
+        This must be a method because a model may have properties that are
+        of type self, this must run after the class is loaded
+
+        Returns
+            openapi_types (dict): The key is attribute name
+                and the value is attribute type.
+        """
+        return {
+            'reason': (str,),  # noqa: E501
+        }
+
+    @cached_property
+    def discriminator():
+        return None
+
+
+    attribute_map = {
+        'reason': 'reason',  # noqa: E501
+    }
+
+    _composed_schemas = {}
+
+    required_properties = set([
+        '_data_store',
+        '_check_type',
+        '_spec_property_naming',
+        '_path_to_item',
+        '_configuration',
+        '_visited_composed_classes',
+    ])
+
+    @convert_js_args_to_python_args
+    def __init__(self, *args, **kwargs):  # noqa: E501
+        """InlineResponse403 - a model defined in OpenAPI
+
+        Keyword Args:
+            _check_type (bool): if True, values for parameters in openapi_types
+                                will be type checked and a TypeError will be
+                                raised if the wrong type is input.
+                                Defaults to True
+            _path_to_item (tuple/list): This is a list of keys or values to
+                                drill down to the model in received_data
+                                when deserializing a response
+            _spec_property_naming (bool): True if the variable names in the input data
+                                are serialized names, as specified in the OpenAPI document.
+                                False if the variable names in the input data
+                                are pythonic names, e.g. snake case (default)
+            _configuration (Configuration): the instance to use when
+                                deserializing a file_type parameter.
+                                If passed, type conversion is attempted
+                                If omitted no type conversion is done.
+            _visited_composed_classes (tuple): This stores a tuple of
+                                classes that we have traveled through so that
+                                if we see that class again we will not use its
+                                discriminator again.
+                                When traveling through a discriminator, the
+                                composed schema that is
+                                is traveled through is added to this set.
+                                For example if Animal has a discriminator
+                                petType and we pass in "Dog", and the class Dog
+                                allOf includes Animal, we move through Animal
+                                once using the discriminator, and pick Dog.
+                                Then in Dog, we will make an instance of the
+                                Animal class but this time we won't travel
+                                through its discriminator because we passed in
+                                _visited_composed_classes = (Animal,)
+            reason (str): [optional]  # noqa: E501
+        """
+
+        _check_type = kwargs.pop('_check_type', True)
+        _spec_property_naming = kwargs.pop('_spec_property_naming', False)
+        _path_to_item = kwargs.pop('_path_to_item', ())
+        _configuration = kwargs.pop('_configuration', None)
+        _visited_composed_classes = kwargs.pop('_visited_composed_classes', ())
+
+        if args:
+            raise ApiTypeError(
+                "Invalid positional arguments=%s passed to %s. Remove those invalid positional arguments." % (
+                    args,
+                    self.__class__.__name__,
+                ),
+                path_to_item=_path_to_item,
+                valid_classes=(self.__class__,),
+            )
+
+        self._data_store = {}
+        self._check_type = _check_type
+        self._spec_property_naming = _spec_property_naming
+        self._path_to_item = _path_to_item
+        self._configuration = _configuration
+        self._visited_composed_classes = _visited_composed_classes + (self.__class__,)
+
+        for var_name, var_value in kwargs.items():
+            if var_name not in self.attribute_map and \
+                        self._configuration is not None and \
+                        self._configuration.discard_unknown_keys and \
+                        self.additional_properties_type is None:
+                # discard variable.
+                continue
+            setattr(self, var_name, var_value)

{paasta_tools-1.23.0 → paasta_tools-1.23.2}/paasta_tools/paastaapi/models/__init__.py

@@ -30,6 +30,7 @@ from paasta_tools.paastaapi.model.inline_object import InlineObject
 from paasta_tools.paastaapi.model.inline_response200 import InlineResponse200
 from paasta_tools.paastaapi.model.inline_response2001 import InlineResponse2001
 from paasta_tools.paastaapi.model.inline_response202 import InlineResponse202
+from paasta_tools.paastaapi.model.inline_response403 import InlineResponse403
 from paasta_tools.paastaapi.model.instance_bounce_status import InstanceBounceStatus
 from paasta_tools.paastaapi.model.instance_mesh_status import InstanceMeshStatus
 from paasta_tools.paastaapi.model.instance_status import InstanceStatus

{paasta_tools-1.23.0 → paasta_tools-1.23.2/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: paasta-tools
-Version: 1.23.0
+Version: 1.23.2
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com

{paasta_tools-1.23.0 → paasta_tools-1.23.2}/paasta_tools.egg-info/SOURCES.txt

@@ -263,6 +263,7 @@ paasta_tools/paastaapi/model/inline_object.py
 paasta_tools/paastaapi/model/inline_response200.py
 paasta_tools/paastaapi/model/inline_response2001.py
 paasta_tools/paastaapi/model/inline_response202.py
+paasta_tools/paastaapi/model/inline_response403.py
 paasta_tools/paastaapi/model/instance_bounce_status.py
 paasta_tools/paastaapi/model/instance_mesh_status.py
 paasta_tools/paastaapi/model/instance_status.py

{paasta_tools-1.23.0 → paasta_tools-1.23.2}/tests/test_kubernetes_tools.py

@@ -4844,6 +4844,80 @@ def test_ensure_service_account_existing():
         mock_client.rbac.create_namespaced_role_binding.assert_not_called()
 
 
+def test_ensure_service_account_existing_different_role():
+    old_iam_role = "arn:aws:iam::000000000000:role/some_role"
+    new_iam_role = "arn:aws:iam::000000000000:role/some-role"
+    namespace = "test_namespace"
+    k8s_role = "mega-admin"
+    expected_sa_name = "paasta--arn-aws-iam-000000000000-role-some-role--mega-admin"
+    with mock.patch(
+        "paasta_tools.kubernetes_tools.kube_config.load_kube_config", autospec=True
+    ), mock.patch(
+        "paasta_tools.kubernetes_tools.KubeClient",
+        autospec=False,
+    ) as mock_kube_client:
+        mock_client = mock.Mock()
+        mock_client.core = mock.Mock(spec=kube_client.CoreV1Api)
+        mock_client.rbac = mock.Mock(spec=kube_client.RbacAuthorizationV1Api)
+        mock_client.core.list_namespaced_service_account.return_value = mock.Mock(
+            spec=V1ServiceAccountList
+        )
+        mock_client.core.list_namespaced_service_account.return_value.items = [
+            V1ServiceAccount(
+                kind="ServiceAccount",
+                metadata=V1ObjectMeta(
+                    name=expected_sa_name,
+                    namespace=namespace,
+                    annotations={"eks.amazonaws.com/role-arn": old_iam_role},
+                ),
+            )
+        ]
+        mock_client.rbac.list_namespaced_role_binding.return_value = mock.Mock(
+            spec=V1RoleBinding,
+        )
+        mock_client.rbac.list_namespaced_role_binding.return_value.items = [
+            V1RoleBinding(
+                kind="ServiceAccount",
+                metadata=V1ObjectMeta(
+                    name=expected_sa_name,
+                    namespace=namespace,
+                ),
+                role_ref=V1RoleRef(
+                    api_group="rbac.authorization.k8s.io",
+                    kind="Role",
+                    name=k8s_role,
+                ),
+                subjects=[
+                    V1Subject(
+                        kind="ServiceAccount",
+                        namespace=namespace,
+                        name=expected_sa_name,
+                    )
+                ],
+            )
+        ]
+        mock_kube_client.return_value = mock_client
+        ensure_service_account(
+            new_iam_role,
+            namespace=namespace,
+            kube_client=mock_client,
+            k8s_role=k8s_role,
+        )
+        mock_client.core.create_namespaced_service_account.assert_not_called()
+        mock_client.core.patch_namespaced_service_account.assert_called_once_with(
+            namespace=namespace,
+            body=V1ServiceAccount(
+                kind="ServiceAccount",
+                metadata=V1ObjectMeta(
+                    name=expected_sa_name,
+                    namespace=namespace,
+                    annotations={"eks.amazonaws.com/role-arn": new_iam_role},
+                ),
+            ),
+            name=expected_sa_name,
+        )
+
+
 def test_ensure_service_account_existing_create_rb_only():
     iam_role = "arn:aws:iam::000000000000:role/some_role"
     namespace = "test_namespace"