paasta-tools 1.4.29__tar.gz → 1.4.31__tar.gz

{paasta-tools-1.4.29/paasta_tools.egg-info → paasta-tools-1.4.31}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.4.29
+Version: 1.4.31
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.4.29 → paasta-tools-1.4.31}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.4.29"
+__version__ = "1.4.31"

{paasta-tools-1.4.29 → paasta-tools-1.4.31}/paasta_tools/cli/schemas/tron_schema.json

@@ -231,6 +231,9 @@
                         }
                     }
                 },
+                "uses_bulkdata": {
+                    "type": "boolean"
+                },
                 "extra_volumes": {
                     "type": "array",
                     "items": {
@@ -662,7 +665,8 @@
                     "type": "string"
                 },
                 "uses_bulkdata": {
-                    "type": "boolean"
+                    "type": "boolean",
+                    "$comment": "XXX: this entry should be deleted once we've refactored existing configs to use the correct toggle location"
                 }
             }
         }

{paasta-tools-1.4.29 → paasta-tools-1.4.31}/paasta_tools/tron_tools.py

@@ -1021,7 +1021,13 @@ def format_tron_action_dict(action_config: TronActionConfig):
         # service account token volumes for service authentication
         result["projected_sa_volumes"] = action_config.get_projected_sa_volumes()
 
-        extra_volumes = action_config.get_extra_volumes()
+        # XXX: now that we're actually passing through extra_volumes correctly (e.g., using get_volumes()),
+        # we can get rid of the default_volumes from the Tron master config
+        system_paasta_config = load_system_paasta_config()
+        extra_volumes = action_config.get_volumes(
+            system_paasta_config.get_volumes(),
+            uses_bulkdata_default=system_paasta_config.get_uses_bulkdata_default(),
+        )
         if executor == "spark":
             is_mrjob = action_config.config_dict.get("mrjob", False)
             system_paasta_config = load_system_paasta_config()

{paasta-tools-1.4.29 → paasta-tools-1.4.31}/paasta_tools/vitesscluster_tools.py

@@ -134,6 +134,7 @@ class GatewayConfigDict(TypedDict, total=False):
     extraEnv: List[Union[KVEnvVar, KVEnvVarValueFrom]]
     extraFlags: Dict[str, str]
     extraLabels: Dict[str, str]
+    lifecycle: Dict[str, Dict[str, Dict[str, List[str]]]]
     replicas: int
     resources: Dict[str, Any]
     annotations: Mapping[str, Any]
@@ -236,6 +237,7 @@ def get_cell_config(
     labels: Dict[str, str],
     node_affinity: dict,
     annotations: Mapping[str, Any],
+    aws_region: str,
 ) -> CellConfigDict:
     """
     get vtgate config
@@ -256,6 +258,26 @@ def get_cell_config(
     config = CellConfigDict(
         name=cell,
         gateway=GatewayConfigDict(
+            lifecycle={
+                "preStop": {
+                    "exec": {
+                        "command": [
+                            "/bin/sh",
+                            "-c",
+                            f"/cloudmap/scripts/deregister_from_cloudmap.sh vtgate-{cell} {aws_region}",
+                        ]
+                    }
+                },
+                "postStart": {
+                    "exec": {
+                        "command": [
+                            "/bin/sh",
+                            "-c",
+                            f"/cloudmap/scripts/register_to_cloudmap.sh vtgate-{cell} {aws_region}",
+                        ]
+                    }
+                },
+            },
             affinity={"nodeAffinity": node_affinity},
             extraEnv=updated_vtgate_extra_env,
             extraFlags={
@@ -523,7 +545,10 @@ def get_keyspaces_config(
         tablet_types = load_system_paasta_config().get_vitess_tablet_types()
         for tablet_type in tablet_types:
             ecosystem = region.split("-")[-1]
-            host = f"mysql-{cluster}-{tablet_type}.dre-{ecosystem}"
+            if tablet_type == "primary":
+                host = f"mysql-{cluster}.dre-{ecosystem}"
+            else:
+                host = f"mysql-{cluster}-{tablet_type}.dre-{ecosystem}"
 
             # We use migration_replication delay for migration tablets and read_replication_delay for everything else
             # Also throttling threshold for refresh and sanitized primaries is set at 30 seconds and everything else at 3 seconds
@@ -712,6 +737,10 @@ class VitessDeploymentConfig(KubernetesDeploymentConfig):
             sys.exit(1)
         return region
 
+    def get_aws_region(self) -> str:
+        region = self.get_region()
+        return f"{region[:2]}-{region[2:6]}-{region[6:7]}"
+
     def get_global_lock_server(self) -> Dict[str, Dict[str, str]]:
         zk_address = self.config_dict.get("zk_address")
         return {
@@ -725,6 +754,7 @@ class VitessDeploymentConfig(KubernetesDeploymentConfig):
     def get_cells(self) -> List[CellConfigDict]:
         cells = self.config_dict.get("cells")
         region = self.get_region()
+        aws_region = self.get_aws_region()
         vtgate_resources = self.config_dict.get("vtgate_resources")
 
         formatted_env = self.get_env_variables()
@@ -741,6 +771,7 @@ class VitessDeploymentConfig(KubernetesDeploymentConfig):
                 labels,
                 node_affinity,
                 annotations,
+                aws_region,
             )
             for cell in cells
         ]

{paasta-tools-1.4.29 → paasta-tools-1.4.31/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.4.29
+Version: 1.4.31
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp