paasta-tools 1.14.3__tar.gz → 1.15.1__tar.gz

{paasta-tools-1.14.3/paasta_tools.egg-info → paasta-tools-1.15.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.14.3
+Version: 1.15.1
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.14.3 → paasta-tools-1.15.1}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.14.3"
+__version__ = "1.15.1"

{paasta-tools-1.14.3 → paasta-tools-1.15.1}/paasta_tools/check_flink_services_health.py

@@ -17,6 +17,8 @@ Usage: ./check_flink_services_health.py [options]
 """
 import datetime
 import logging
+from typing import Dict
+from typing import List
 from typing import Sequence
 from typing import Tuple
 
@@ -27,7 +29,6 @@ from paasta_tools import flinkeks_tools
 from paasta_tools.check_services_replication_tools import main
 from paasta_tools.check_services_replication_tools import parse_args
 from paasta_tools.flink_tools import FlinkDeploymentConfig
-from paasta_tools.kubernetes_tools import filter_pods_by_service_instance
 from paasta_tools.kubernetes_tools import is_pod_ready
 from paasta_tools.kubernetes_tools import V1Pod
 from paasta_tools.monitoring_tools import check_under_replication
@@ -132,14 +133,12 @@ def get_cr_name(si_pods: Sequence[V1Pod]) -> str:
 
 def check_flink_service_health(
     instance_config: FlinkDeploymentConfig,
-    all_pods: Sequence[V1Pod],
+    pods_by_service_instance: Dict[str, Dict[str, List[V1Pod]]],
     replication_checker: KubeSmartstackEnvoyReplicationChecker,
     dry_run: bool = False,
 ) -> None:
-    si_pods = filter_pods_by_service_instance(
-        pod_list=all_pods,
-        service=instance_config.service,
-        instance=instance_config.instance,
+    si_pods = pods_by_service_instance.get(instance_config.service, {}).get(
+        instance_config.instance, []
     )
     taskmanagers_expected_cnt = instance_config.config_dict.get(
         "taskmanager", {"instances": 10}

{paasta-tools-1.14.3 → paasta-tools-1.15.1}/paasta_tools/check_kubernetes_services_replication.py

@@ -31,8 +31,9 @@ CRITICAL. If replication_threshold is defined in the yelpsoa config for a servic
 instance then it will be used instead.
 """
 import logging
+from typing import Dict
+from typing import List
 from typing import Optional
-from typing import Sequence
 from typing import Union
 
 from paasta_tools import eks_tools
@@ -41,7 +42,6 @@ from paasta_tools import monitoring_tools
 from paasta_tools.check_services_replication_tools import main
 from paasta_tools.check_services_replication_tools import parse_args
 from paasta_tools.eks_tools import EksDeploymentConfig
-from paasta_tools.kubernetes_tools import filter_pods_by_service_instance
 from paasta_tools.kubernetes_tools import is_pod_ready
 from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig
 from paasta_tools.kubernetes_tools import V1Pod
@@ -56,14 +56,13 @@ DEFAULT_ALERT_AFTER = "10m"
 def check_healthy_kubernetes_tasks_for_service_instance(
     instance_config: Union[KubernetesDeploymentConfig, EksDeploymentConfig],
     expected_count: int,
-    all_pods: Sequence[V1Pod],
+    pods_by_service_instance: Dict[str, Dict[str, List[V1Pod]]],
     dry_run: bool = False,
 ) -> None:
-    si_pods = filter_pods_by_service_instance(
-        pod_list=all_pods,
-        service=instance_config.service,
-        instance=instance_config.instance,
+    si_pods = pods_by_service_instance.get(instance_config.service, {}).get(
+        instance_config.instance, []
     )
+
     num_healthy_tasks = len([pod for pod in si_pods if is_pod_ready(pod)])
     log.info(
         f"Checking {instance_config.service}.{instance_config.instance} in kubernetes as it is not in smartstack"
@@ -78,7 +77,7 @@ def check_healthy_kubernetes_tasks_for_service_instance(
 
 def check_kubernetes_pod_replication(
     instance_config: Union[KubernetesDeploymentConfig, EksDeploymentConfig],
-    all_pods: Sequence[V1Pod],
+    pods_by_service_instance: Dict[str, Dict[str, List[V1Pod]]],
     replication_checker: KubeSmartstackEnvoyReplicationChecker,
     dry_run: bool = False,
 ) -> Optional[bool]:
@@ -126,7 +125,7 @@ def check_kubernetes_pod_replication(
         check_healthy_kubernetes_tasks_for_service_instance(
             instance_config=instance_config,
             expected_count=expected_count,
-            all_pods=all_pods,
+            pods_by_service_instance=pods_by_service_instance,
             dry_run=dry_run,
         )
         return None

{paasta-tools-1.14.3 → paasta-tools-1.15.1}/paasta_tools/check_services_replication_tools.py

@@ -14,12 +14,16 @@
 import argparse
 import logging
 import sys
+from multiprocessing import Pool
+from os import cpu_count
 from typing import Any
 from typing import Callable
-from typing import Container
+from typing import cast
+from typing import Dict
 from typing import List
 from typing import Optional
 from typing import Sequence
+from typing import Set
 from typing import Tuple
 from typing import Type
 
@@ -29,6 +33,7 @@ from mypy_extensions import NamedArg
 from paasta_tools.kubernetes_tools import get_all_managed_namespaces
 from paasta_tools.kubernetes_tools import get_all_nodes
 from paasta_tools.kubernetes_tools import get_all_pods
+from paasta_tools.kubernetes_tools import group_pods_by_service_instance
 from paasta_tools.kubernetes_tools import KubeClient
 from paasta_tools.kubernetes_tools import V1Node
 from paasta_tools.kubernetes_tools import V1Pod
@@ -52,7 +57,7 @@ log = logging.getLogger(__name__)
 CheckServiceReplication = Callable[
     [
         Arg(InstanceConfig_T, "instance_config"),
-        Arg(Sequence[V1Pod], "all_pods"),
+        Arg(Dict[str, Dict[str, List[V1Pod]]], "pods_by_service_instance"),
         Arg(Any, "replication_checker"),
         NamedArg(bool, "dry_run"),
     ],
@@ -102,16 +107,6 @@ def parse_args() -> argparse.Namespace:
         dest="dry_run",
         help="Print Sensu alert events and metrics instead of sending them",
     )
-    parser.add_argument(
-        "--additional-namespaces",
-        help="full names of namespaces to check services replication for that don't match --namespace-prefix"
-        "Used only when service is kubernetes",
-        dest="additional_namespaces",
-        nargs="+",
-        # we default this to paasta since we always want to run this check on paasta namespace
-        # to avoid having two cron jobs running with two different namespace-prefix
-        default=["paasta"],
-    )
     parser.add_argument(
         "--eks",
         help="This flag checks k8 services running on EKS",
@@ -131,7 +126,7 @@ def check_services_replication(
     instance_type_class: Type[InstanceConfig_T],
     check_service_replication: CheckServiceReplication,
     replication_checker: ReplicationChecker,
-    all_pods: Sequence[V1Pod],
+    pods_by_service_instance: Dict[str, Dict[str, List[V1Pod]]],
     dry_run: bool = False,
 ) -> Tuple[int, int]:
     service_instances_set = set(service_instances)
@@ -151,7 +146,7 @@ def check_services_replication(
             if instance_config.get_docker_image():
                 is_well_replicated = check_service_replication(
                     instance_config=instance_config,
-                    all_pods=all_pods,
+                    pods_by_service_instance=pods_by_service_instance,
                     replication_checker=replication_checker,
                     dry_run=dry_run,
                 )
@@ -211,14 +206,14 @@ def main(
             system_paasta_config=system_paasta_config,
         )
     else:
-        pods, nodes = get_kubernetes_pods_and_nodes(
-            additional_namespaces=args.additional_namespaces,
-        )
+        pods, nodes = get_kubernetes_pods_and_nodes()
         replication_checker = KubeSmartstackEnvoyReplicationChecker(
             nodes=nodes,
             system_paasta_config=system_paasta_config,
         )
 
+    pods_by_service_instance = group_pods_by_service_instance(pods)
+
     count_under_replicated, total = check_services_replication(
         soa_dir=args.soa_dir,
         cluster=cluster,
@@ -226,7 +221,7 @@ def main(
         instance_type_class=instance_type_class,
         check_service_replication=check_service_replication,
         replication_checker=replication_checker,
-        all_pods=pods,
+        pods_by_service_instance=pods_by_service_instance,
         dry_run=args.dry_run,
     )
     pct_under_replicated = 0 if total == 0 else 100 * count_under_replicated / total
@@ -256,21 +251,73 @@ def main(
     sys.exit(exit_code)
 
 
+# XXX: is there a base class for the k8s clientlib models that we could use to type `obj`?
+def set_local_vars_configuration_to_none(obj: Any, visited: Set[int] = None) -> None:
+    """
+    Recursive function to ensure that k8s clientlib objects are pickleable.
+
+    Without this, k8s clientlib objects can't be used by multiprocessing functions
+    as those pickle data to shuttle between processes.
+    """
+    if visited is None:
+        visited = set()
+
+    # Avoid infinite recursion for objects that have already been visited
+    obj_id = id(obj)
+    if obj_id in visited:
+        return
+    visited.add(obj_id)
+
+    # if the object has the attribute, set it to None to essentially delete it
+    if hasattr(obj, "local_vars_configuration"):
+        setattr(obj, "local_vars_configuration", None)
+
+    # recursively check attributes of the object
+    if hasattr(obj, "__dict__"):
+        for attr_name, attr_value in obj.__dict__.items():
+            set_local_vars_configuration_to_none(attr_value, visited)
+
+    # if the object is iterable/a collection, iterate over its elements
+    elif isinstance(obj, (list, tuple, set)):
+        for item in obj:
+            set_local_vars_configuration_to_none(item, visited)
+    elif isinstance(obj, dict):
+        for value in obj.values():
+            set_local_vars_configuration_to_none(value, visited)
+
+
+def __fetch_pods(namespace: str) -> List[V1Pod]:
+    kube_client = KubeClient()
+    pods = get_all_pods(kube_client, namespace)
+    for pod in pods:
+        # this is pretty silly, but V1Pod cannot be pickled otherwise since the local_vars_configuration member
+        # is not picklable - and pretty much every k8s model has this member ;_;
+        set_local_vars_configuration_to_none(pod)
+    return pods
+
+
+def __get_all_pods_parallel(from_namespaces: Set[str]) -> List[V1Pod]:
+    all_pods: List[V1Pod] = []
+    with Pool() as pool:
+        for pod_list in pool.imap_unordered(
+            __fetch_pods,
+            from_namespaces,
+            chunksize=len(from_namespaces) // cast(int, cpu_count()),
+        ):
+            all_pods.extend(pod_list)
+    return all_pods
+
+
 def get_kubernetes_pods_and_nodes(
     namespace: Optional[str] = None,
-    additional_namespaces: Optional[Container[str]] = None,
 ) -> Tuple[List[V1Pod], List[V1Node]]:
     kube_client = KubeClient()
 
-    all_pods: List[V1Pod] = []
     if namespace:
         all_pods = get_all_pods(kube_client=kube_client, namespace=namespace)
     else:
-        all_managed_namespaces = get_all_managed_namespaces(kube_client)
-        for managed_namespace in all_managed_namespaces:
-            all_pods.extend(
-                get_all_pods(kube_client=kube_client, namespace=managed_namespace)
-            )
+        all_managed_namespaces = set(get_all_managed_namespaces(kube_client))
+        all_pods = __get_all_pods_parallel(all_managed_namespaces)
 
     all_nodes = get_all_nodes(kube_client)
 

{paasta-tools-1.14.3 → paasta-tools-1.15.1}/paasta_tools/cli/cmds/spark_run.py

@@ -12,6 +12,7 @@ from typing import Dict
 from typing import List
 from typing import Mapping
 from typing import Optional
+from typing import Set
 from typing import Tuple
 from typing import Union
 
@@ -31,6 +32,7 @@ from paasta_tools.cli.utils import get_service_auth_token
 from paasta_tools.cli.utils import lazy_choices_completer
 from paasta_tools.cli.utils import list_instances
 from paasta_tools.clusterman import get_clusterman_metrics
+from paasta_tools.kubernetes_tools import get_service_account_name
 from paasta_tools.spark_tools import auto_add_timeout_for_spark_job
 from paasta_tools.spark_tools import create_spark_config_str
 from paasta_tools.spark_tools import DEFAULT_SPARK_RUNTIME_TIMEOUT
@@ -38,6 +40,7 @@ from paasta_tools.spark_tools import DEFAULT_SPARK_SERVICE
 from paasta_tools.spark_tools import get_volumes_from_spark_k8s_configs
 from paasta_tools.spark_tools import get_webui_url
 from paasta_tools.spark_tools import inject_spark_conf_str
+from paasta_tools.tron_tools import load_tron_instance_configs
 from paasta_tools.utils import _run
 from paasta_tools.utils import DEFAULT_SOA_DIR
 from paasta_tools.utils import filter_templates_from_config
@@ -419,6 +422,29 @@ def add_subparser(subparsers):
         default=False,
     )
 
+    aws_group.add_argument(
+        "--force-pod-identity",
+        help=(
+            "Normally the spark executor will use the pod identity defined "
+            "for the relevant instance in yelpsoa-configs. If the instance "
+            "isn't setup there yet, you can override the IAM role arn here."
+            " However, it must already be set for a different instance of "
+            "the service. Must be used with --executor-pod-identity."
+        ),
+        default=None,
+    )
+
+    aws_group.add_argument(
+        "--executor-pod-identity",
+        help=(
+            "Launch the executor pod with pod-identity derived from "
+            "the iam_role settings attached to the instance settings in "
+            "SOA configs. See also --force-pod-identity."
+        ),
+        action="store_true",
+        default=False,
+    )
+
     jupyter_group = list_parser.add_argument_group(
         title="Jupyter kernel culling options",
         description="Idle kernels will be culled by default. Idle "
@@ -627,6 +653,19 @@ def get_spark_env(
     return spark_env
 
 
+def get_all_iam_roles_for_service(
+    service: str,
+    cluster: str,
+) -> Set[str]:
+    tron_instance_configs = load_tron_instance_configs(service, cluster)
+    roles = set()
+    for action in tron_instance_configs:
+        role = action.get_iam_role()
+        if role:
+            roles.add(role)
+    return roles
+
+
 def _parse_user_spark_args(
     spark_args: str,
 ) -> Dict[str, str]:
@@ -1205,6 +1244,47 @@ def paasta_spark_run(args: argparse.Namespace) -> int:
         )
         return 1
 
+    service_account_name = None
+    iam_role = instance_config.get_iam_role()
+    if args.executor_pod_identity and not (iam_role or args.force_pod_identity):
+        print(
+            "--executor-pod-identity set but no iam_role settings found.",
+            file=sys.stderr,
+        )
+        return 1
+    if args.executor_pod_identity:
+        if args.force_pod_identity:
+            if args.yelpsoa_config_root != DEFAULT_SOA_DIR:
+                print(
+                    "--force-pod-identity cannot be used with --yelpsoa-config-root",
+                    file=sys.stderr,
+                )
+                return 1
+
+            allowed_iam_roles = get_all_iam_roles_for_service(
+                args.service, args.cluster
+            )
+            if args.force_pod_identity not in allowed_iam_roles:
+                print(
+                    f"{args.force_pod_identity} is not an allowed role for this service. "
+                    f"Allowed roles are: {allowed_iam_roles}.",
+                    file=sys.stderr,
+                )
+                return 1
+
+            service_account_name = get_service_account_name(args.force_pod_identity)
+        else:
+            service_account_name = get_service_account_name(iam_role)
+        if (
+            not args.aws_credentials_yaml
+            and not args.aws_profile
+            and not args.assume_aws_role
+        ):
+            args.aws_credentials_yaml = (
+                system_paasta_config.get_default_spark_iam_user()
+            )
+        log.info(f"Running executor with service account {service_account_name}")
+
     aws_creds = get_aws_credentials(
         service=args.service,
         aws_credentials_yaml=args.aws_credentials_yaml,
@@ -1213,6 +1293,14 @@ def paasta_spark_run(args: argparse.Namespace) -> int:
         session_duration=args.aws_role_duration,
         use_web_identity=args.use_web_identity,
     )
+
+    # If executor pods use a service account, they don't need static aws creds
+    # but the driver still does
+    if service_account_name:
+        executor_aws_creds = None
+    else:
+        executor_aws_creds = aws_creds
+
     docker_image_digest = get_docker_image(args, instance_config)
     if docker_image_digest is None:
         return 1
@@ -1254,11 +1342,12 @@ def paasta_spark_run(args: argparse.Namespace) -> int:
         paasta_service=args.service,
         paasta_instance=paasta_instance,
         extra_volumes=cast(List[Mapping[str, str]], volumes),
-        aws_creds=aws_creds,
+        aws_creds=executor_aws_creds,
         aws_region=args.aws_region,
         force_spark_resource_configs=args.force_spark_resource_configs,
         use_eks=True,
         k8s_server_address=k8s_server_address,
+        service_account_name=service_account_name,
     )
 
     return configure_and_run_docker_container(

{paasta-tools-1.14.3 → paasta-tools-1.15.1}/paasta_tools/cli/schemas/eks_schema.json

@@ -502,6 +502,19 @@
                         },
                         "page": {
                             "type": "boolean"
+                        },
+                        "alert_after": {
+                            "oneOf": [
+                                {
+                                    "type": "string",
+                                    "pattern": "^[1-9]+[0-9]*[YMWDhms]$",
+                                    "$comment": "See https://pysensu-yelp.readthedocs.io/en/latest/#pysensu_yelp.human_to_seconds"
+                                },
+                                {
+                                    "type": "integer",
+                                    "minimum": 0
+                                }
+                            ]
                         }
                     },
                     "additionalProperties": true

{paasta-tools-1.14.3 → paasta-tools-1.15.1}/paasta_tools/cli/schemas/kubernetes_schema.json

@@ -502,6 +502,19 @@
                         },
                         "page": {
                             "type": "boolean"
+                        },
+                        "alert_after": {
+                            "oneOf": [
+                                {
+                                    "type": "string",
+                                    "pattern": "^[1-9]+[0-9]*[YMWDhms]$",
+                                    "$comment": "See https://pysensu-yelp.readthedocs.io/en/latest/#pysensu_yelp.human_to_seconds"
+                                },
+                                {
+                                    "type": "integer",
+                                    "minimum": 0
+                                }
+                            ]
                         }
                     },
                     "additionalProperties": true

{paasta-tools-1.14.3 → paasta-tools-1.15.1}/paasta_tools/cli/schemas/tron_schema.json

@@ -622,7 +622,17 @@
                             "type": "string"
                         },
                         "alert_after": {
-                            "type": "string"
+                            "oneOf": [
+                                {
+                                    "type": "string",
+                                    "pattern": "^[1-9]+[0-9]*[YMWDhms]$",
+                                    "$comment": "See https://pysensu-yelp.readthedocs.io/en/latest/#pysensu_yelp.human_to_seconds"
+                                },
+                                {
+                                    "type": "integer",
+                                    "minimum": 0
+                                }
+                            ]
                         },
                         "check_that_every_day_has_a_successful_run": {
                             "type": "boolean"

{paasta-tools-1.14.3 → paasta-tools-1.15.1}/paasta_tools/envoy_tools.py

@@ -204,6 +204,7 @@ def get_multiple_backends(
     envoy_host: str,
     envoy_admin_port: int,
     envoy_admin_endpoint_format: str,
+    resolve_hostnames: bool = True,
 ) -> Dict[str, List[Tuple[EnvoyBackend, bool]]]:
     """Fetches JSON from Envoy admin's /clusters endpoint and returns a list of backends.
 
@@ -247,11 +248,15 @@ def get_multiple_backends(
                                 casper_endpoint_found = True
                                 continue
 
-                        try:
-                            hostname = socket.gethostbyaddr(address)[0].split(".")[0]
-                        except socket.herror:
-                            # Default to the raw IP address if we can't lookup the hostname
-                            hostname = address
+                        hostname = address
+                        if resolve_hostnames:
+                            try:
+                                hostname = socket.gethostbyaddr(address)[0].split(".")[
+                                    0
+                                ]
+                            except socket.herror:
+                                # Default to the raw IP address if we can't lookup the hostname
+                                pass
 
                         cluster_backends.append(
                             (
@@ -352,6 +357,7 @@ def get_replication_for_all_services(
         envoy_host=envoy_host,
         envoy_admin_port=envoy_admin_port,
         envoy_admin_endpoint_format=envoy_admin_endpoint_format,
+        resolve_hostnames=False,  # we're not really going to use the hostnames, so skip fetching them to save time
     )
     return collections.Counter(
         [

{paasta-tools-1.14.3 → paasta-tools-1.15.1}/paasta_tools/kubernetes/bin/paasta_cleanup_remote_run_resources.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import argparse
+import logging
 from datetime import datetime
 from datetime import timedelta
 from datetime import timezone
@@ -30,15 +31,19 @@ from paasta_tools.kubernetes_tools import KubeClient
 
 ListingFuncType = Callable[[KubeClient, str], Sequence[Any]]
 DeletionFuncType = Callable[[str, str], Any]
+log = logging.getLogger(__name__)
 
 
-def clean_namespace(kube_client: KubeClient, namespace: str, age_limit: datetime):
+def clean_namespace(
+    kube_client: KubeClient, namespace: str, age_limit: datetime, dry_run: bool = False
+):
     """Clean ephemeral remote-run resource in a namespace
 
     :param KubeClient kube_client: kubernetes client
     :param str namepsace: kubernetes namespace
     :param datetime age_limit: expiration time for resources
     """
+    dry_run_msg = " (dry_run)" if dry_run else ""
     cleanup_actions: Sequence[Tuple[DeletionFuncType, ListingFuncType]] = (
         (
             kube_client.core.delete_namespaced_service_account,
@@ -54,7 +59,9 @@ def clean_namespace(kube_client: KubeClient, namespace: str, age_limit: datetime
                 or entity.metadata.creation_timestamp > age_limit
             ):
                 continue
-            delete_func(entity.metadata.name, namespace)
+            log.info(f"Deleting {entity.metadata.name} in {namespace}{dry_run_msg}")
+            if not dry_run:
+                delete_func(entity.metadata.name, namespace)
 
 
 def parse_args() -> argparse.Namespace:
@@ -68,15 +75,30 @@ def parse_args() -> argparse.Namespace:
         default=600,
         help="Maximum age, in seconds, resources are allowed to have",
     )
+    parser.add_argument(
+        "-n",
+        "--dry-run",
+        action="store_true",
+        default=False,
+        help="Do not actually delete resources",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        default=False,
+        help="More verbose logging",
+    )
     return parser.parse_args()
 
 
 def main():
     args = parse_args()
+    logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
     kube_client = KubeClient()
     age_limit = datetime.now(tz=timezone.utc) - timedelta(seconds=args.max_age)
     for namespace in get_all_managed_namespaces(kube_client):
-        clean_namespace(kube_client, namespace, age_limit)
+        clean_namespace(kube_client, namespace, age_limit, args.dry_run)
 
 
 if __name__ == "__main__":

{paasta-tools-1.14.3 → paasta-tools-1.15.1}/paasta_tools/kubernetes/remote_run.py

@@ -372,7 +372,7 @@ def create_pod_scoped_role(
             labels={POD_OWNER_LABEL: user},
         ),
     )
-    kube_client.core.create_namespaced_role(namespace=namespace, body=role)
+    kube_client.rbac.create_namespaced_role(namespace=namespace, body=role)
     return role_name
 
 

{paasta-tools-1.14.3 → paasta-tools-1.15.1}/paasta_tools/kubernetes_tools.py

@@ -3439,8 +3439,11 @@ def get_pods_by_node(kube_client: KubeClient, node: V1Node) -> Sequence[V1Pod]:
     ).items
 
 
-def get_all_pods(kube_client: KubeClient, namespace: str) -> List[V1Pod]:
-    return kube_client.core.list_namespaced_pod(namespace=namespace).items
+def get_all_pods(kube_client: KubeClient, namespace: Optional[str]) -> List[V1Pod]:
+    if namespace:
+        return kube_client.core.list_namespaced_pod(namespace=namespace).items
+    else:
+        return kube_client.core.list_pod_for_all_namespaces().items
 
 
 @time_cache(ttl=300)
@@ -3461,6 +3464,26 @@ def filter_pods_by_service_instance(
     ]
 
 
+def group_pods_by_service_instance(
+    pods: Sequence[V1Pod],
+) -> Dict[str, Dict[str, List[V1Pod]]]:
+    pods_by_service_instance: Dict[str, Dict[str, List[V1Pod]]] = {}
+    for pod in pods:
+        if pod.metadata.labels is not None:
+            service = pod.metadata.labels.get("paasta.yelp.com/service")
+            instance = pod.metadata.labels.get("paasta.yelp.com/instance")
+
+            if service and instance:
+                if service not in pods_by_service_instance:
+                    pods_by_service_instance[service] = {}
+                if instance not in pods_by_service_instance[service]:
+                    pods_by_service_instance[service][instance] = []
+
+                pods_by_service_instance[service][instance].append(pod)
+
+    return pods_by_service_instance
+
+
 def _is_it_ready(
     it: Union[V1Pod, V1Node],
 ) -> bool:

{paasta-tools-1.14.3 → paasta-tools-1.15.1}/paasta_tools/utils.py

@@ -2044,6 +2044,7 @@ class SystemPaastaConfigDict(TypedDict, total=False):
     uses_bulkdata_default: bool
     enable_automated_redeploys_default: bool
     enable_tron_tsc: bool
+    default_spark_iam_user: str
 
 
 def load_system_paasta_config(
@@ -2142,6 +2143,11 @@ class SystemPaastaConfig:
     def get_spark_use_eks_default(self) -> bool:
         return self.config_dict.get("spark_use_eks_default", False)
 
+    def get_default_spark_iam_user(self) -> str:
+        return self.config_dict.get(
+            "default_spark_iam_user", "/etc/boto_cfg/mrjob.yaml"
+        )
+
     def get_sidecar_requirements_config(
         self,
     ) -> Dict[str, KubeContainerResourceRequest]: