paasta-tools 1.4.13__tar.gz → 1.4.15__tar.gz

{paasta-tools-1.4.13/paasta_tools.egg-info → paasta-tools-1.4.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.4.13
+Version: 1.4.15
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.4.13 → paasta-tools-1.4.15}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.4.13"
+__version__ = "1.4.15"

{paasta-tools-1.4.13 → paasta-tools-1.4.15}/paasta_tools/cli/cmds/spark_run.py

@@ -72,8 +72,6 @@ DEFAULT_DRIVER_MEMORY_BY_SPARK = "1g"
 # Extra room for memory overhead and for any other running inside container
 DOCKER_RESOURCE_ADJUSTMENT_FACTOR = 2
 
-DEFAULT_AWS_PROFILE = "default"
-
 DEPRECATED_OPTS = {
     "j": "spark.jars",
     "jars": "spark.jars",
@@ -361,7 +359,6 @@ def add_subparser(subparsers):
         "--aws-credentials-yaml is not specified and --service is either "
         "not specified or the service does not have credentials in "
         "/etc/boto_cfg",
-        default=DEFAULT_AWS_PROFILE,
     )
 
     aws_group.add_argument(
@@ -1062,16 +1059,15 @@ def update_args_from_tronfig(args: argparse.Namespace) -> Optional[Dict[str, str
             if field_name == "spark_args":
                 value = " ".join([f"{k}={v}" for k, v in dict(value).items()])
 
-            # Befutify for printing
+            # Beautify for printing
             arg_name_str = (f"--{arg_name.replace('_', '-')}").ljust(31, " ")
-            field_name_str = field_name.ljust(28)
 
             # Only load iam_role value if --aws-profile is not set
-            if field_name == "iam_role" and args.aws_profile != DEFAULT_AWS_PROFILE:
+            if field_name == "iam_role" and args.aws_profile is not None:
                 print(
                     PaastaColors.yellow(
-                        f"Overwriting args with Tronfig: {arg_name_str} => {field_name_str} : IGNORE, "
-                        "since --aws-profile is provided"
+                        f"Ignoring Tronfig: `{field_name} : {value}`, since `--aws-profile` is provided. "
+                        f"We are giving higher priority to `--aws-profile` in case of paasta spark-run adhoc runs."
                     ),
                 )
                 continue
@@ -1079,7 +1075,7 @@ def update_args_from_tronfig(args: argparse.Namespace) -> Optional[Dict[str, str
             if hasattr(args, arg_name):
                 print(
                     PaastaColors.yellow(
-                        f"Overwriting args with Tronfig: {arg_name_str} => {field_name_str} : {value}"
+                        f"Overwriting args with Tronfig: {arg_name_str} => {field_name} : {value}"
                     ),
                 )
             setattr(args, arg_name, value)

{paasta-tools-1.4.13 → paasta-tools-1.4.15}/paasta_tools/setup_tron_namespace.py

@@ -23,6 +23,7 @@ file for it, and send the updated file to Tron.
 import argparse
 import logging
 import sys
+from typing import List
 
 import ruamel.yaml as yaml
 
@@ -31,7 +32,9 @@ from paasta_tools import tron_tools
 from paasta_tools.kubernetes_tools import ensure_service_account
 from paasta_tools.kubernetes_tools import KubeClient
 from paasta_tools.tron_tools import KUBERNETES_NAMESPACE
+from paasta_tools.tron_tools import load_tron_service_config
 from paasta_tools.tron_tools import MASTER_NAMESPACE
+from paasta_tools.tron_tools import TronJobConfig
 from paasta_tools.utils import load_system_paasta_config
 
 log = logging.getLogger(__name__)
@@ -68,32 +71,29 @@ def parse_args():
     return args
 
 
-def ensure_service_accounts(raw_config: str) -> None:
+def ensure_service_accounts(job_configs: List[TronJobConfig]) -> None:
     # NOTE: these are lru_cache'd so it should be fine to call these for every service
     system_paasta_config = load_system_paasta_config()
     kube_client = KubeClient()
-    # this is kinda silly, but the tron create_config functions return strings
-    # we should refactor to pass the dicts around until the we're going to send the config to tron
-    # (where we can finally convert it to a string)
-    config = yaml.safe_load(raw_config)
-    for _, job in config.get("jobs", {}).items():
-        for _, action in job.get("actions", {}).items():
-            if action.get("service_account_name") is not None:
+
+    for job in job_configs:
+        for action in job.get_actions():
+            if action.get_iam_role():
                 ensure_service_account(
-                    action["service_account_name"],
+                    action.get_iam_role(),
                     namespace=KUBERNETES_NAMESPACE,
                     kube_client=kube_client,
                 )
                 # spark executors are special in that we want the SA to exist in two namespaces:
                 # the tron namespace - for the spark driver
                 # and the spark namespace - for the spark executor
-                if action.get("executor") == "spark":
+                if action.get_executor() == "spark":
                     # this kubeclient creation is lru_cache'd so it should be fine to call this for every spark action
                     spark_kube_client = KubeClient(
                         config_file=system_paasta_config.get_spark_kubeconfig()
                     )
                     ensure_service_account(
-                        action["service_account_name"],
+                        action.get_iam_role(),
                         namespace=spark_tools.SPARK_EXECUTOR_NAMESPACE,
                         kube_client=spark_kube_client,
                     )
@@ -171,7 +171,15 @@ def main():
             else:
                 # PaaSTA will not necessarily have created the SAs we want to use
                 # ...so let's go ahead and create them!
-                ensure_service_accounts(new_config)
+                job_configs = load_tron_service_config(
+                    service=service,
+                    cluster=args.cluster,
+                    load_deployments=False,
+                    soa_dir=args.soa_dir,
+                    # XXX: we can remove for_validation now that we've refactored how service account stuff works
+                    for_validation=False,
+                )
+                ensure_service_accounts(job_configs)
 
                 if client.update_namespace(service, new_config):
                     updated.append(service)

{paasta-tools-1.4.13 → paasta-tools-1.4.15/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.4.13
+Version: 1.4.15
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.4.13 → paasta-tools-1.4.15}/paasta_tools.egg-info/requires.txt

@@ -47,7 +47,7 @@ requests-cache>=0.4.10
 retry
 ruamel.yaml
 sensu-plugin
-service-configuration-lib>=2.18.17
+service-configuration-lib>=2.18.19
 signalfx
 slackclient>=1.2.1
 sticht>=1.1.0

{paasta-tools-1.4.13 → paasta-tools-1.4.15}/requirements-minimal.txt

@@ -54,7 +54,7 @@ requests-cache >= 0.4.10
 retry
 ruamel.yaml
 sensu-plugin
-service-configuration-lib >= 2.18.17
+service-configuration-lib >= 2.18.19
 signalfx
 slackclient >= 1.2.1
 sticht >= 1.1.0