paasta-tools 1.4.2__tar.gz → 1.4.4__tar.gz

{paasta-tools-1.4.2/paasta_tools.egg-info → paasta-tools-1.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.4.2
+Version: 1.4.4
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.4.2 → paasta-tools-1.4.4}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.4.2"
+__version__ = "1.4.4"

{paasta-tools-1.4.2 → paasta-tools-1.4.4}/paasta_tools/cli/cmds/logs.py

@@ -1195,12 +1195,10 @@ class VectorLogsReader(LogReader):
         stream_name = get_log_name_for_service(service, prefix="app_output")
         superregion = self.get_superregion_for_cluster(clusters[0])
         reader = S3LogsReader(superregion)
-        start_date = start_time.date()
-        end_date = end_time.date()
         aggregated_logs: List[Dict[str, Any]] = []
 
         for line in reader.get_log_reader(
-            log_name=stream_name, min_date=start_date, max_date=end_date
+            log_name=stream_name, start_datetime=start_time, end_datetime=end_time
         ):
             if paasta_log_line_passes_filter(
                 line,

{paasta-tools-1.4.2 → paasta-tools-1.4.4}/paasta_tools/setup_kubernetes_cr.py

@@ -49,11 +49,15 @@ from paasta_tools.utils import get_config_hash
 from paasta_tools.utils import get_git_sha_from_dockerurl
 from paasta_tools.utils import load_all_configs
 from paasta_tools.utils import load_system_paasta_config
+from paasta_tools.vitesscluster_tools import load_vitess_service_instance_configs
 
 
 log = logging.getLogger(__name__)
 
 
+INSTANCE_TYPE_TO_CONFIG_LOADER = {"vitesscluster": load_vitess_service_instance_configs}
+
+
 class StdoutKubeClient:
     """Replace all destructive operations in Kubernetes APIs with
     writing out YAML to stdout."""
@@ -352,6 +356,13 @@ def reconcile_kubernetes_resource(
                 load_deployments=True,
                 soa_dir=DEFAULT_SOA_DIR,
             )
+            if crd.file_prefix in INSTANCE_TYPE_TO_CONFIG_LOADER:
+                config = INSTANCE_TYPE_TO_CONFIG_LOADER[crd.file_prefix](
+                    service=service,
+                    instance=inst,
+                    cluster=cluster,
+                    soa_dir=DEFAULT_SOA_DIR,
+                )
             git_sha = get_git_sha_from_dockerurl(soa_config.get_docker_url(), long=True)
             formatted_resource = format_custom_resource(
                 instance_config=config,

{paasta-tools-1.4.2 → paasta-tools-1.4.4}/paasta_tools/spark_tools.py

@@ -6,7 +6,6 @@ from typing import Any
 from typing import cast
 from typing import Dict
 from typing import List
-from typing import Literal
 from typing import Mapping
 from typing import Set
 
@@ -24,7 +23,6 @@ SPARK_DRIVER_POOL = "stable"
 SPARK_JOB_USER = "TRON"
 SPARK_PROMETHEUS_SHARD = "ml-compute"
 SPARK_DNS_POD_TEMPLATE = "/nail/srv/configs/spark_dns_pod_template.yaml"
-MEM_MULTIPLIER = {"k": 1024, "m": 1024**2, "g": 1024**3, "t": 1024**4}
 
 log = logging.getLogger(__name__)
 
@@ -249,22 +247,3 @@ def get_spark_driver_monitoring_labels(
         "spark.yelp.com/driver_ui_port": ui_port_str,
     }
     return labels
-
-
-def get_spark_memory_in_unit(mem: str, unit: Literal["k", "m", "g", "t"]) -> float:
-    """
-    Converts Spark memory to the desired unit.
-    mem is the same format as JVM memory strings: just number or number followed by 'k', 'm', 'g' or 't'.
-    unit can be 'k', 'm', 'g' or 't'.
-    Returns memory as a float converted to the desired unit.
-    """
-    try:
-        memory_bytes = float(mem)
-    except ValueError:
-        try:
-            memory_bytes = float(mem[:-1]) * MEM_MULTIPLIER[mem[-1]]
-        except (ValueError, IndexError):
-            print(f"Unable to parse memory value {mem}. Defaulting to 2 GB.")
-            memory_bytes = 2147483648  # default to 2 GB
-    memory_unit = memory_bytes / MEM_MULTIPLIER[unit]
-    return memory_unit

{paasta-tools-1.4.2 → paasta-tools-1.4.4}/paasta_tools/tron_tools.py

@@ -282,35 +282,6 @@ class TronActionConfig(InstanceConfig):
         self.job, self.action = decompose_instance(instance)
         # Indicate whether this config object is created for validation
         self.for_validation = for_validation
-        self.action_spark_config = None
-        if self.get_executor() == "spark":
-            # build the complete Spark configuration
-            # TODO: add conditional check for Spark specific commands spark-submit, pyspark etc ?
-            self.action_spark_config = self.build_spark_config()
-
-    def get_cpus(self) -> float:
-        # set Spark driver pod CPU if it is specified by Spark arguments
-        if (
-            self.action_spark_config
-            and "spark.driver.cores" in self.action_spark_config
-        ):
-            return float(self.action_spark_config["spark.driver.cores"])
-        # we fall back to this default if there's no spark.driver.cores config
-        return super().get_cpus()
-
-    def get_mem(self) -> float:
-        # set Spark driver pod memory if it is specified by Spark arguments
-        if (
-            self.action_spark_config
-            and "spark.driver.memory" in self.action_spark_config
-        ):
-            return int(
-                spark_tools.get_spark_memory_in_unit(
-                    self.action_spark_config["spark.driver.memory"], "m"
-                )
-            )
-        # we fall back to this default if there's no spark.driver.memory config
-        return super().get_mem()
 
     def build_spark_config(self) -> Dict[str, str]:
         system_paasta_config = load_system_paasta_config()
@@ -469,6 +440,7 @@ class TronActionConfig(InstanceConfig):
         system_paasta_config: Optional["SystemPaastaConfig"] = None,
     ) -> Dict[str, str]:
         env = super().get_env(system_paasta_config=system_paasta_config)
+
         if self.get_executor() == "spark":
             # Required by some sdks like boto3 client. Throws NoRegionError otherwise.
             # AWS_REGION takes precedence if set.
@@ -633,20 +605,6 @@ class TronActionConfig(InstanceConfig):
             error_msgs.append(
                 f"{self.get_job_name()}.{self.get_action_name()} must have a deploy_group set"
             )
-        # We are not allowing users to specify `cpus` and `mem` configuration if the action is a Spark job
-        # with driver running on k8s (executor: spark), because we derive these values from `spark.driver.cores`
-        # and `spark.driver.memory` in order to avoid confusion.
-        if self.get_executor() == "spark":
-            if "cpus" in self.config_dict:
-                error_msgs.append(
-                    f"{self.get_job_name()}.{self.get_action_name()} is a Spark job. `cpus` config is not allowed. "
-                    f"Please specify the driver cores using `spark.driver.cores`."
-                )
-            if "mem" in self.config_dict:
-                error_msgs.append(
-                    f"{self.get_job_name()}.{self.get_action_name()} is a Spark job. `mem` config is not allowed. "
-                    f"Please specify the driver memory using `spark.driver.memory`."
-                )
         return error_msgs
 
     def get_pool(self) -> str:
@@ -1017,26 +975,21 @@ def format_tron_action_dict(action_config: TronActionConfig):
         if executor == "spark":
             is_mrjob = action_config.config_dict.get("mrjob", False)
             system_paasta_config = load_system_paasta_config()
-            # inject additional Spark configs in case of Spark commands
+            # inject spark configs to the original spark-submit command
+            spark_config = action_config.build_spark_config()
             result["command"] = spark_tools.build_spark_command(
                 result["command"],
-                action_config.action_spark_config,
+                spark_config,
                 is_mrjob,
                 action_config.config_dict.get(
                     "max_runtime", spark_tools.DEFAULT_SPARK_RUNTIME_TIMEOUT
                 ),
             )
-            # point to the KUBECONFIG needed by Spark driver
             result["env"]["KUBECONFIG"] = system_paasta_config.get_spark_kubeconfig()
-
             # spark, unlike normal batches, needs to expose several ports for things like the spark
             # ui and for executor->driver communication
             result["ports"] = list(
-                set(
-                    spark_tools.get_spark_ports_from_config(
-                        action_config.action_spark_config
-                    )
-                )
+                set(spark_tools.get_spark_ports_from_config(spark_config))
             )
             # mount KUBECONFIG file for Spark drivers to communicate with EKS cluster
             extra_volumes.append(
@@ -1050,12 +1003,10 @@ def format_tron_action_dict(action_config: TronActionConfig):
             )
             # Add pod annotations and labels for Spark monitoring metrics
             monitoring_annotations = (
-                spark_tools.get_spark_driver_monitoring_annotations(
-                    action_config.action_spark_config
-                )
+                spark_tools.get_spark_driver_monitoring_annotations(spark_config)
             )
             monitoring_labels = spark_tools.get_spark_driver_monitoring_labels(
-                action_config.action_spark_config
+                spark_config
             )
             result["annotations"].update(monitoring_annotations)
             result["labels"].update(monitoring_labels)

{paasta-tools-1.4.2 → paasta-tools-1.4.4}/paasta_tools/utils.py

@@ -2022,6 +2022,12 @@ class SystemPaastaConfigDict(TypedDict, total=False):
     use_multiple_log_readers: Optional[List[str]]
     service_auth_token_settings: ProjectedSAVolume
     always_authenticating_services: List[str]
+    mysql_port_mappings: Dict
+    vitess_images: Dict
+    superregion_to_region_mapping: Dict
+    vitess_tablet_types: List[str]
+    vitess_tablet_pool_type_mapping: Dict
+    vitess_throttling_config: Dict
 
 
 def load_system_paasta_config(
@@ -2733,6 +2739,52 @@ class SystemPaastaConfig:
     def get_always_authenticating_services(self) -> List[str]:
         return self.config_dict.get("always_authenticating_services", [])
 
+    def get_mysql_port_mappings(self) -> Dict:
+        return self.config_dict.get("mysql_port_mappings", {})
+
+    def get_vitess_images(self) -> Dict:
+        return self.config_dict.get(
+            "vitess_images",
+            {
+                "vtctld_image": "docker-paasta.yelpcorp.com:443/vitess_base:v16.0.3",
+                "vtgate_image": "docker-paasta.yelpcorp.com:443/vitess_base:v16.0.3",
+                "vttablet_image": "docker-paasta.yelpcorp.com:443/vitess_base:v16.0.3",
+                "vtadmin_image": "docker-paasta.yelpcorp.com:443/vtadmin:v16.0.3",
+            },
+        )
+
+    def get_superregion_to_region_mapping(self) -> Dict:
+        return self.config_dict.get("superregion_to_region_mapping", {})
+
+    def get_vitess_tablet_types(self) -> List:
+        return self.config_dict.get("vitess_tablet_types", ["primary", "migration"])
+
+    def get_vitess_tablet_pool_type_mapping(self) -> Dict:
+        return self.config_dict.get("vitess_tablet_pool_type_mapping", {})
+
+    def get_vitess_throttling_config(self) -> Dict:
+        return self.config_dict.get(
+            "vitess_throttling_config",
+            {
+                "migration": {
+                    "throttle_query_table": "migration_replication_delay",
+                    "throttle_metrics_threshold": "7200",
+                },
+                "read": {
+                    "throttle_query_table": "read_replication_delay",
+                    "throttle_metrics_threshold": "3",
+                },
+                "reporting": {
+                    "throttle_query_table": "reporting_replication_delay",
+                    "throttle_metrics_threshold": "7200",
+                },
+                "primary": {
+                    "throttle_query_table": "read_replication_delay",
+                    "throttle_metrics_threshold": "3",
+                },
+            },
+        )
+
 
 def _run(
     command: Union[str, List[str]],