paasta-tools 1.6.4__tar.gz → 1.7.0__tar.gz

{paasta-tools-1.6.4/paasta_tools.egg-info → paasta-tools-1.7.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.6.4
+Version: 1.7.0
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.6.4 → paasta-tools-1.7.0}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.6.4"
+__version__ = "1.7.0"

{paasta-tools-1.6.4 → paasta-tools-1.7.0}/paasta_tools/cli/cmds/logs.py

@@ -329,6 +329,12 @@ def paasta_app_output_passes_filter(
     # https://github.com/gweis/isodate/issues/53
     except ValueError:
         return True
+    except AttributeError:
+        # Timestamp might be missing. We had an issue where OTel was splitting overly long log lines
+        # and not including timestamps in the resulting log records (OBSPLAT-2216).
+        # Although this was then fixed in OTel, we should not rely on timestamps being present,
+        # as the format cannot be guaranteed.
+        return False
     if not check_timestamp_in_range(timestamp, start_time, end_time):
         return False
     return (

{paasta-tools-1.6.4 → paasta-tools-1.7.0}/paasta_tools/cli/cmds/spark_run.py

@@ -239,9 +239,9 @@ def add_subparser(subparsers):
             "default_pool"
         )
     except PaastaNotConfiguredError:
-        default_spark_cluster = "pnw-devc"
+        default_spark_cluster = "pnw-devc-spark"
         default_spark_pool = "batch"
-        valid_clusters = ["spark-pnw-prod", "pnw-devc"]
+        valid_clusters = ["pnw-devc-spark", "pnw-prod-spark"]
 
     list_parser.add_argument(
         "-c",

{paasta-tools-1.6.4 → paasta-tools-1.7.0}/paasta_tools/contrib/service_shard_update.py

@@ -39,6 +39,13 @@ def parse_args():
         action="store_true",
         dest="verbose",
     )
+    parser.add_argument(
+        "-d",
+        "--dry-run",
+        help="Do not commit changes to git",
+        action="store_true",
+        dest="dry_run",
+    )
     parser.add_argument(
         "--source-id",
         help="String to attribute the changes in the commit message.",
@@ -115,6 +122,48 @@ def parse_args():
         type=int,
         dest="timeout_server_ms",
     )
+    parser.add_argument(
+        "--autotune-min-cpus",
+        help="Minimum number of CPUs Autotune should give the shard",
+        required=False,
+        type=float,
+        dest="autotune_min_cpus",
+    )
+    parser.add_argument(
+        "--autotune-max-cpus",
+        help="Maximum number of CPUs Autotune should give the shard",
+        required=False,
+        type=float,
+        dest="autotune_max_cpus",
+    )
+    parser.add_argument(
+        "--autotune-min-mem",
+        help="Minimum amount of memory Autotune should give the shard",
+        required=False,
+        type=int,
+        dest="autotune_min_mem",
+    )
+    parser.add_argument(
+        "--autotune-max-mem",
+        help="Maximum amount of memory Autotune should give the shard",
+        required=False,
+        type=int,
+        dest="autotune_max_mem",
+    )
+    parser.add_argument(
+        "--autotune-min-disk",
+        help="Minimum amount of disk Autotune should give the shard",
+        required=False,
+        type=int,
+        dest="autotune_min_disk",
+    )
+    parser.add_argument(
+        "--autotune-max-disk",
+        help="Maximum amount of disk Autotune should give the shard",
+        required=False,
+        type=int,
+        dest="autotune_max_disk",
+    )
     return parser.parse_args()
 
 
@@ -195,9 +244,11 @@ def main(args):
                     instance_config = {
                         "deploy_group": f"{deploy_prefix}.{args.shard_name}",
                         "min_instances": args.min_instance_count,
-                        "max_instances": args.prod_max_instance_count
-                        if deploy_prefix == "prod"
-                        else args.non_prod_max_instance_count,
+                        "max_instances": (
+                            args.prod_max_instance_count
+                            if deploy_prefix == "prod"
+                            else args.non_prod_max_instance_count
+                        ),
                         "env": {
                             "PAASTA_SECRET_BUGSNAG_API_KEY": "SECRET(bugsnag_api_key)",
                         },
@@ -217,6 +268,40 @@ def main(args):
                         instance_config["cpus"] = args.cpus
                     if args.mem is not None:
                         instance_config["mem"] = args.mem
+                    if any(
+                        (
+                            args.autotune_min_cpus,
+                            args.autotune_max_cpus,
+                            args.autotune_min_mem,
+                            args.autotune_max_mem,
+                            args.autotune_min_disk,
+                            args.autotune_max_disk,
+                        )
+                    ):
+                        limit_config = {}
+                        limit_config["cpus"] = {
+                            "min": args.autotune_min_cpus,
+                            "max": args.autotune_max_cpus,
+                        }
+                        limit_config["mem"] = {
+                            "min": args.autotune_min_mem,
+                            "max": args.autotune_max_mem,
+                        }
+                        limit_config["disk"] = {
+                            "min": args.autotune_min_disk,
+                            "max": args.autotune_max_disk,
+                        }
+
+                        # remove any None values to keep the config clean
+                        for resource in list(limit_config):
+                            for key in list(limit_config[resource]):
+                                if limit_config[resource][key] is None:
+                                    del limit_config[resource][key]
+                            if len(limit_config[resource]) == 0:
+                                del limit_config[resource]
+
+                        if len(limit_config) > 0:
+                            instance_config["autotune_limits"] = limit_config
                     # If the service config does not contain definitions for the shard in each ecosystem
                     # Add the missing definition and write to the corresponding config
                     if args.shard_name not in config_file.keys():
@@ -247,7 +332,7 @@ def main(args):
             log.info(f"{args.shard_name} is in smartstack config already, skipping.")
 
         # Only commit to remote if changes were made
-        if changes_made:
+        if changes_made and not args.dry_run:
             updater.commit_to_remote()
             trigger_deploys(args.service)
         else:

{paasta-tools-1.6.4 → paasta-tools-1.7.0}/paasta_tools/setup_tron_namespace.py

@@ -224,7 +224,7 @@ def main():
             # since we need to print out what failed in either case
             failed.append(service)
 
-    if args.bulk_config_fetch:
+    if args.dry_run and args.bulk_config_fetch:
         updated_namespaces = client.update_namespaces(new_configs)
 
         if updated_namespaces:

{paasta-tools-1.6.4 → paasta-tools-1.7.0}/paasta_tools/spark_tools.py

@@ -20,7 +20,7 @@ DEFAULT_SPARK_RUNTIME_TIMEOUT = "12h"
 SPARK_AWS_CREDS_PROVIDER = "com.amazonaws.auth.WebIdentityTokenCredentialsProvider"
 SPARK_EXECUTOR_NAMESPACE = "paasta-spark"
 SPARK_DRIVER_POOL = "stable"
-SPARK_JOB_USER = "TRON"
+SPARK_TRON_JOB_USER = "TRON"
 SPARK_PROMETHEUS_SHARD = "ml-compute"
 SPARK_DNS_POD_TEMPLATE = "/nail/srv/configs/spark_dns_pod_template.yaml"
 MEM_MULTIPLIER = {"k": 1024, "m": 1024**2, "g": 1024**3, "t": 1024**4}
@@ -176,7 +176,9 @@ def inject_spark_conf_str(original_cmd: str, spark_conf_str: str) -> str:
     return original_cmd
 
 
-def auto_add_timeout_for_spark_job(cmd: str, timeout_job_runtime: str) -> str:
+def auto_add_timeout_for_spark_job(
+    cmd: str, timeout_job_runtime: str, silent: bool = False
+) -> str:
     # Timeout only to be added for spark-submit commands
     # TODO: Add timeout for jobs using mrjob with spark-runner
     if "spark-submit" not in cmd:
@@ -189,16 +191,17 @@ def auto_add_timeout_for_spark_job(cmd: str, timeout_job_runtime: str) -> str:
             split_cmd = cmd.split("spark-submit")
             # split_cmd[0] will always be an empty string or end with a space
             cmd = f"{split_cmd[0]}timeout {timeout_job_runtime} spark-submit{split_cmd[1]}"
-            log.info(
-                PaastaColors.blue(
-                    f"NOTE: Job will exit in given time {timeout_job_runtime}. "
-                    f"Adjust timeout value using --timeout-job-timeout. "
-                    f"New Updated Command with timeout: {cmd}"
-                ),
-            )
+            if not silent:
+                log.info(
+                    PaastaColors.blue(
+                        f"NOTE: Job will exit in given time {timeout_job_runtime}. "
+                        f"Adjust timeout value using --timeout-job-runtime. "
+                        f"New Updated Command with timeout: {cmd}"
+                    ),
+                )
     except Exception as e:
         err_msg = (
-            f"'timeout' could not be added to command: '{cmd}' due to error '{e}'. "
+            f"'timeout' could not be added to spark command: '{cmd}' due to error '{e}'. "
             "Please report to #spark."
         )
         log.warn(err_msg)
@@ -211,9 +214,12 @@ def build_spark_command(
     spark_config_dict: Dict[str, Any],
     is_mrjob: bool,
     timeout_job_runtime: str,
+    silent: bool = False,
 ) -> str:
-    command = f"{inject_spark_conf_str(original_cmd, create_spark_config_str(spark_config_dict, is_mrjob=is_mrjob))}"
-    return auto_add_timeout_for_spark_job(command, timeout_job_runtime)
+    command = inject_spark_conf_str(
+        original_cmd, create_spark_config_str(spark_config_dict, is_mrjob=is_mrjob)
+    )
+    return auto_add_timeout_for_spark_job(command, timeout_job_runtime, silent=silent)
 
 
 def get_spark_ports_from_config(spark_conf: Dict[str, str]) -> List[int]:
@@ -238,6 +244,7 @@ def get_spark_driver_monitoring_annotations(
 
 def get_spark_driver_monitoring_labels(
     spark_config: Dict[str, str],
+    user: str,
 ) -> Dict[str, str]:
     """
     Returns Spark driver pod labels - generally for Prometheus metric relabeling.
@@ -245,7 +252,7 @@ def get_spark_driver_monitoring_labels(
     ui_port_str = str(spark_config.get("spark.ui.port", ""))
     labels = {
         "paasta.yelp.com/prometheus_shard": SPARK_PROMETHEUS_SHARD,
-        "spark.yelp.com/user": SPARK_JOB_USER,
+        "spark.yelp.com/user": user,
         "spark.yelp.com/driver_ui_port": ui_port_str,
     }
     return labels

{paasta-tools-1.6.4 → paasta-tools-1.7.0}/paasta_tools/tron_tools.py

@@ -347,7 +347,7 @@ class TronActionConfig(InstanceConfig):
 
         docker_img_url = self.get_docker_url(system_paasta_config)
 
-        spark_conf_builder = SparkConfBuilder()
+        spark_conf_builder = SparkConfBuilder(is_driver_on_k8s_tron=True)
         spark_conf = spark_conf_builder.get_spark_conf(
             cluster_manager="kubernetes",
             spark_app_base_name=spark_app_name,
@@ -366,7 +366,7 @@ class TronActionConfig(InstanceConfig):
             force_spark_resource_configs=self.config_dict.get(
                 "force_spark_resource_configs", False
             ),
-            user=spark_tools.SPARK_JOB_USER,
+            user=spark_tools.SPARK_TRON_JOB_USER,
         )
         # delete the dynamically generated spark.app.id to prevent frequent config updates in Tron.
         # spark.app.id will be generated later by yelp spark-submit wrapper or Spark itself.
@@ -380,16 +380,17 @@ class TronActionConfig(InstanceConfig):
                 if "spark.app.name" not in stringified_spark_args
                 else stringified_spark_args["spark.app.name"]
             )
-        # TODO: Remove this once dynamic pod template is generated inside the driver using spark-submit wrapper
+
+        # TODO(MLCOMPUTE-1220): Remove this once dynamic pod template is generated inside the driver using spark-submit wrapper
         if "spark.kubernetes.executor.podTemplateFile" in spark_conf:
-            print(
+            log.info(
                 f"Replacing spark.kubernetes.executor.podTemplateFile="
                 f"{spark_conf['spark.kubernetes.executor.podTemplateFile']} with "
                 f"spark.kubernetes.executor.podTemplateFile={spark_tools.SPARK_DNS_POD_TEMPLATE}"
             )
-            spark_conf[
-                "spark.kubernetes.executor.podTemplateFile"
-            ] = spark_tools.SPARK_DNS_POD_TEMPLATE
+        spark_conf[
+            "spark.kubernetes.executor.podTemplateFile"
+        ] = spark_tools.SPARK_DNS_POD_TEMPLATE
 
         spark_conf.update(
             {
@@ -1044,6 +1045,7 @@ def format_tron_action_dict(action_config: TronActionConfig):
                 action_config.config_dict.get(
                     "max_runtime", spark_tools.DEFAULT_SPARK_RUNTIME_TIMEOUT
                 ),
+                silent=True,
             )
             # point to the KUBECONFIG needed by Spark driver
             result["env"]["KUBECONFIG"] = system_paasta_config.get_spark_kubeconfig()
@@ -1074,7 +1076,8 @@ def format_tron_action_dict(action_config: TronActionConfig):
                 )
             )
             monitoring_labels = spark_tools.get_spark_driver_monitoring_labels(
-                action_config.action_spark_config
+                action_config.action_spark_config,
+                user=spark_tools.SPARK_TRON_JOB_USER,
             )
             result["annotations"].update(monitoring_annotations)
             result["labels"].update(monitoring_labels)

{paasta-tools-1.6.4 → paasta-tools-1.7.0/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.6.4
+Version: 1.7.0
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp