paasta-tools 1.16.2__tar.gz → 1.16.3__tar.gz

{paasta-tools-1.16.2/paasta_tools.egg-info → paasta-tools-1.16.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.16.2
+Version: 1.16.3
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.16.2 → paasta-tools-1.16.3}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.16.2"
+__version__ = "1.16.3"

{paasta-tools-1.16.2 → paasta-tools-1.16.3}/paasta_tools/cli/cmds/spark_run.py

@@ -6,6 +6,7 @@ import re
 import shlex
 import socket
 import sys
+from configparser import ConfigParser
 from typing import Any
 from typing import cast
 from typing import Dict
@@ -84,6 +85,14 @@ DEPRECATED_OPTS = {
 
 SPARK_COMMANDS = {"pyspark", "spark-submit"}
 
+# config looks as follows:
+# [default]
+# aws_access_key_id = ...
+# aws_secret_access_key = ...
+SPARK_DRIVER_IAM_USER = (
+    "/nail/etc/spark_driver_k8s_role_assumer/spark_driver_k8s_role_assumer.ini"
+)
+
 log = logging.getLogger(__name__)
 
 
@@ -154,6 +163,11 @@ def add_subparser(subparsers):
         help=argparse.SUPPRESS,
         action=DeprecatedAction,
     )
+    list_parser.add_argument(
+        "--get-eks-token-via-iam-user",
+        help="Use IAM user to get EKS token for long running spark-run jobs",
+        action="store_true",
+    )
 
     group = list_parser.add_mutually_exclusive_group()
     group.add_argument(
@@ -648,7 +662,22 @@ def get_spark_env(
         spark_env["SPARK_DAEMON_CLASSPATH"] = "/opt/spark/extra_jars/*"
         spark_env["SPARK_NO_DAEMONIZE"] = "true"
 
-    spark_env["KUBECONFIG"] = system_paasta_config.get_spark_kubeconfig()
+    if args.get_eks_token_via_iam_user:
+        with open(SPARK_DRIVER_IAM_USER) as f:
+            config = ConfigParser()
+            config.read_file(f)
+
+        # these env variables are consumed by a script specified in the spark kubeconfig - and which will result in a tightly-scoped IAM identity being used for EKS cluster access
+        spark_env["GET_EKS_TOKEN_AWS_ACCESS_KEY_ID"] = config["default"][
+            "aws_access_key_id"
+        ]
+        spark_env["GET_EKS_TOKEN_AWS_SECRET_ACCESS_KEY"] = config["default"][
+            "aws_secret_access_key"
+        ]
+
+        spark_env["KUBECONFIG"] = system_paasta_config.get_spark_iam_user_kubeconfig()
+    else:
+        spark_env["KUBECONFIG"] = system_paasta_config.get_spark_kubeconfig()
 
     return spark_env
 
@@ -838,9 +867,9 @@ def configure_and_run_docker_container(
     if pod_template_path:
         volumes.append(f"{pod_template_path}:{pod_template_path}:rw")
 
-    volumes.append(
-        f"{system_paasta_config.get_spark_kubeconfig()}:{system_paasta_config.get_spark_kubeconfig()}:ro"
-    )
+    # NOTE: we mount a directory here since the kubeconfig we're transitioning to requires a helper script that will co-exist in the same directory
+    kubeconfig_dir = os.path.dirname(system_paasta_config.get_spark_kubeconfig())
+    volumes.append(f"{kubeconfig_dir}:{kubeconfig_dir}:ro")
 
     environment = instance_config.get_env_dictionary()  # type: ignore
     spark_conf_str = create_spark_config_str(spark_conf, is_mrjob=args.mrjob)
@@ -1148,6 +1177,12 @@ def update_args_from_tronfig(args: argparse.Namespace) -> Optional[Dict[str, str
 
 
 def paasta_spark_run(args: argparse.Namespace) -> int:
+    if args.get_eks_token_via_iam_user and os.getuid() != 0:
+        print("Re-executing paasta spark-run with sudo..", file=sys.stderr)
+        # argv[0] is treated as command name, so prepending "sudo"
+        os.execvp("sudo", ["sudo"] + sys.argv)
+        return  # will not reach unless above function is mocked
+
     driver_envs_from_tronfig: Dict[str, str] = dict()
     if args.tronfig is not None:
         if args.job_id is None:

{paasta-tools-1.16.2 → paasta-tools-1.16.3}/paasta_tools/utils.py

@@ -2036,6 +2036,7 @@ class SystemPaastaConfigDict(TypedDict, total=False):
     skip_cpu_burst_validation: List[str]
     tron_default_pool_override: str
     spark_kubeconfig: str
+    spark_iam_user_kubeconfig: str
     kube_clusters: Dict
     spark_use_eks_default: bool
     sidecar_requirements_config: Dict[str, KubeContainerResourceRequest]
@@ -2758,6 +2759,11 @@ class SystemPaastaConfig:
     def get_spark_kubeconfig(self) -> str:
         return self.config_dict.get("spark_kubeconfig", "/etc/kubernetes/spark.conf")
 
+    def get_spark_iam_user_kubeconfig(self) -> str:
+        return self.config_dict.get(
+            "spark_iam_user_kubeconfig", "/etc/kubernetes/spark2.conf"
+        )
+
     def get_kube_clusters(self) -> Dict:
         return self.config_dict.get("kube_clusters", {})
 

{paasta-tools-1.16.2 → paasta-tools-1.16.3/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.16.2
+Version: 1.16.3
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp