paasta-tools 1.15.1__tar.gz → 1.16.0__tar.gz

{paasta-tools-1.15.1/paasta_tools.egg-info → paasta-tools-1.16.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.15.1
+Version: 1.16.0
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.15.1 → paasta-tools-1.16.0}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.15.1"
+__version__ = "1.16.0"

{paasta-tools-1.15.1 → paasta-tools-1.16.0}/paasta_tools/tron_tools.py

@@ -697,13 +697,14 @@ class TronActionConfig(InstanceConfig):
         override this value. To control this, we have an optional config item that we'll puppet onto Tron masters
         which this function will read.
         """
-        return (
-            self.config_dict.get(
+        if self.get_executor() == "spark":
+            pool = load_system_paasta_config().get_default_spark_driver_pool_override()
+        else:
+            pool = self.config_dict.get(
                 "pool", load_system_paasta_config().get_tron_default_pool_override()
             )
-            if not self.get_executor() == "spark"
-            else spark_tools.SPARK_DRIVER_POOL
-        )
+
+        return pool
 
     def get_spark_executor_pool(self) -> str:
         return self.config_dict.get("pool", DEFAULT_SPARK_EXECUTOR_POOL)

{paasta-tools-1.15.1 → paasta-tools-1.16.0}/paasta_tools/utils.py

@@ -123,6 +123,10 @@ DEFAULT_CPU_BURST_ADD = 1
 
 DEFAULT_SOA_CONFIGS_GIT_URL = "sysgit.yelpcorp.com"
 
+# To ensure the Spark driver not being interrupted due to spot instances,
+# we use stable pool for drivers
+DEFAULT_SPARK_DRIVER_POOL = "stable"
+
 log = logging.getLogger(__name__)
 log.addHandler(logging.NullHandler())
 
@@ -2045,6 +2049,7 @@ class SystemPaastaConfigDict(TypedDict, total=False):
     enable_automated_redeploys_default: bool
     enable_tron_tsc: bool
     default_spark_iam_user: str
+    default_spark_driver_pool_override: str
 
 
 def load_system_paasta_config(
@@ -2148,6 +2153,17 @@ class SystemPaastaConfig:
             "default_spark_iam_user", "/etc/boto_cfg/mrjob.yaml"
         )
 
+    def get_default_spark_driver_pool_override(self) -> str:
+        """
+        If defined, fetches the override for what pool to run a Spark driver in.
+        Otherwise, returns the default Spark driver pool.
+
+        :returns: The default_spark_driver_pool_override specified in the paasta configuration
+        """
+        return self.config_dict.get(
+            "default_spark_driver_pool_override", DEFAULT_SPARK_DRIVER_POOL
+        )
+
     def get_sidecar_requirements_config(
         self,
     ) -> Dict[str, KubeContainerResourceRequest]:

{paasta-tools-1.15.1 → paasta-tools-1.16.0/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.15.1
+Version: 1.16.0
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp