paasta-tools 1.4.17__tar.gz → 1.4.19__tar.gz

{paasta-tools-1.4.17/paasta_tools.egg-info → paasta-tools-1.4.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.4.17
+Version: 1.4.19
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.4.17 → paasta-tools-1.4.19}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.4.17"
+__version__ = "1.4.19"

{paasta-tools-1.4.17 → paasta-tools-1.4.19}/paasta_tools/cli/schemas/tron_schema.json

@@ -383,18 +383,18 @@
                         },
                         "spark.executor.memory": {
                             "type": "string",
-                            "pattern": "^[1-9]+[0-9]*[kmg]$"
+                            "pattern": "^[1-9]+[0-9]*[kmgt]$"
                         },
                         "spark.driver.memory": {
                             "type": "string",
-                            "pattern": "^[1-9]+[0-9]*[kmg]$"
+                            "pattern": "^[1-9]+[0-9]*[kmgt]$"
                         },
                         "spark.driver.memoryOverhead": {
                             "$comment": "we still need to validate this in code since there's a spark-enforced minimum of 384mb",
                             "oneOf": [
                                 {
                                     "type": "string",
-                                    "pattern": "^[1-9]+[0-9]*[kmg]$"
+                                    "pattern": "^[1-9]+[0-9]*[kmgt]$"
                                 },
                                 {
                                     "type": "number",
@@ -408,7 +408,7 @@
                             "oneOf": [
                                 {
                                     "type": "string",
-                                    "pattern": "^[1-9]+[0-9]*[kmg]$"
+                                    "pattern": "^[1-9]+[0-9]*[kmgt]$"
                                 },
                                 {
                                     "type": "number",
@@ -419,7 +419,7 @@
                         },
                         "spark.driver.maxResultSize": {
                             "type": "string",
-                            "pattern": "^[1-9]+[0-9]*[kmg]$"
+                            "pattern": "^[1-9]+[0-9]*[kmgt]$"
                         },
                         "spark.scheduler.minRegisteredResourcesRatio": {
                             "type": "number",
@@ -429,7 +429,7 @@
                         },
                         "spark.scheduler.maxRegisteredResourcesWaitingTime": {
                             "type": "string",
-                            "pattern": "^[1-9]+[0-9]*(s|min)$"
+                            "pattern": "^[1-9]+[0-9]*(ms|s|m|min|h|d)$"
                         },
                         "spark.kubernetes.allocation.batch.size": {
                             "type": "integer"

{paasta-tools-1.4.17 → paasta-tools-1.4.19}/paasta_tools/kubernetes_tools.py

@@ -531,8 +531,8 @@ def limit_size_with_hash(name: str, limit: int = 63, suffix: int = 4) -> str:
     """
     if len(name) > limit:
         digest = hashlib.md5(name.encode()).digest()
-        hash = base64.b32encode(digest).decode().replace("=", "").lower()
-        return f"{name[:(limit-suffix-1)]}-{hash[:suffix]}"
+        hashed = base64.b32encode(digest).decode().replace("=", "").lower()
+        return f"{name[:(limit-suffix-1)]}-{hashed[:suffix]}"
     else:
         return name
 

{paasta-tools-1.4.17 → paasta-tools-1.4.19}/paasta_tools/long_running_service_tools.py

@@ -397,7 +397,7 @@ class LongRunningServiceConfig(InstanceConfig):
         return error_messages
 
     def get_bounce_margin_factor(self) -> float:
-        return self.config_dict.get("bounce_margin_factor", 1.0)
+        return self.config_dict.get("bounce_margin_factor", 0.95)
 
     def get_should_ping_for_unhealthy_pods(self, default: bool) -> bool:
         return self.config_dict.get("should_ping_for_unhealthy_pods", default)

{paasta-tools-1.4.17 → paasta-tools-1.4.19}/paasta_tools/tron_tools.py

@@ -364,6 +364,18 @@ class TronActionConfig(InstanceConfig):
             ),
             user=spark_tools.SPARK_JOB_USER,
         )
+        # delete the dynamically generated spark.app.id to prevent frequent config updates in Tron.
+        # spark.app.id will be generated later by yelp spark-submit wrapper or Spark itself.
+        spark_conf.pop("spark.app.id", None)
+        # use a static spark.app.name to prevent frequent config updates in Tron.
+        # md5 and base64 will always generate the same encoding for a string.
+        # This spark.app.name might be overridden by yelp spark-submit wrapper.
+        if "spark.app.name" in spark_conf:
+            spark_conf["spark.app.name"] = limit_size_with_hash(
+                f"tron_spark_{self.get_service()}_{self.get_instance()}_{self.get_action_name()}"
+                if "spark.app.name" not in stringified_spark_args
+                else stringified_spark_args["spark.app.name"]
+            )
         # TODO: Remove this once dynamic pod template is generated inside the driver using spark-submit wrapper
         if "spark.kubernetes.executor.podTemplateFile" in spark_conf:
             print(

{paasta-tools-1.4.17 → paasta-tools-1.4.19/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.4.17
+Version: 1.4.19
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp