paasta-tools 1.4.21__tar.gz → 1.4.23__tar.gz

{paasta-tools-1.4.21/paasta_tools.egg-info → paasta-tools-1.4.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.4.21
+Version: 1.4.23
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.4.21 → paasta-tools-1.4.23}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.4.21"
+__version__ = "1.4.23"

{paasta-tools-1.4.21 → paasta-tools-1.4.23}/paasta_tools/cli/schemas/tron_schema.json

@@ -418,8 +418,17 @@
                             ]
                         },
                         "spark.driver.maxResultSize": {
-                            "type": "string",
-                            "pattern": "^[1-9]+[0-9]*[kmgt]$"
+                            "oneOf": [
+                                {
+                                    "type": "string",
+                                    "pattern": "^[1-9]+[0-9]*[kmgt]$"
+                                },
+                                {
+                                    "type": "number",
+                                    "minimum": 0,
+                                    "exclusiveMinimum": true
+                                }
+                            ]
                         },
                         "spark.scheduler.minRegisteredResourcesRatio": {
                             "type": "number",
@@ -650,6 +659,9 @@
                 },
                 "deploy_group": {
                     "type": "string"
+                },
+                "uses_bulkdata": {
+                    "type": "boolean"
                 }
             }
         }

{paasta-tools-1.4.21 → paasta-tools-1.4.23}/paasta_tools/frameworks/native_service_config.py

@@ -171,7 +171,8 @@ class NativeServiceConfig(LongRunningServiceConfig):
         computed separately.
         """
         docker_volumes = self.get_volumes(
-            system_volumes=system_paasta_config.get_volumes()
+            system_volumes=system_paasta_config.get_volumes(),
+            uses_bulkdata_default=system_paasta_config.get_uses_bulkdata_default(),
         )
         task: TaskInfo = {
             "name": "",

{paasta-tools-1.4.21 → paasta-tools-1.4.23}/paasta_tools/kubernetes_tools.py

@@ -2142,7 +2142,8 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
             service=self.service, namespace=self.get_nerve_namespace()
         )
         docker_volumes = self.get_volumes(
-            system_volumes=system_paasta_config.get_volumes()
+            system_volumes=system_paasta_config.get_volumes(),
+            uses_bulkdata_default=system_paasta_config.get_uses_bulkdata_default(),
         )
 
         hacheck_sidecar_volumes = system_paasta_config.get_hacheck_sidecar_volumes()

{paasta-tools-1.4.21 → paasta-tools-1.4.23}/paasta_tools/tron_tools.py

@@ -357,7 +357,10 @@ class TronActionConfig(InstanceConfig):
             paasta_service=self.get_service(),
             paasta_instance=self.get_instance(),
             docker_img=docker_img_url,
-            extra_volumes=self.get_volumes(system_paasta_config.get_volumes()),
+            extra_volumes=self.get_volumes(
+                system_paasta_config.get_volumes(),
+                uses_bulkdata_default=system_paasta_config.get_uses_bulkdata_default(),
+            ),
             use_eks=True,
             k8s_server_address=get_k8s_url_for_cluster(self.get_cluster()),
             force_spark_resource_configs=self.config_dict.get(

{paasta-tools-1.4.21 → paasta-tools-1.4.23}/paasta_tools/utils.py

@@ -624,7 +624,8 @@ class InstanceConfig:
 
         Defaults to 1024 (1GiB) if no value is specified in the config.
 
-        :returns: The amount of disk space specified by the config, 1024 MiB if not specified"""
+        :returns: The amount of disk space specified by the config, 1024 MiB if not specified
+        """
         disk = self.config_dict.get("disk", default)
         return disk
 
@@ -979,7 +980,8 @@ class InstanceConfig:
 
         Eventually this may be implemented with Mesos roles, once a framework can register under multiple roles.
 
-        :returns: the "pool" attribute in your config dict, or the string "default" if not specified."""
+        :returns: the "pool" attribute in your config dict, or the string "default" if not specified.
+        """
         return self.config_dict.get("pool", "default")
 
     def get_pool_constraints(self) -> List[Constraint]:
@@ -998,17 +1000,17 @@ class InstanceConfig:
         """
         return self.config_dict.get("net", "bridge")
 
-    def has_bulkdata(
-        self,
-    ) -> bool:
-        return self.config_dict.get("uses_bulkdata", True)
-
-    def get_volumes(self, system_volumes: Sequence[DockerVolume]) -> List[DockerVolume]:
+    def get_volumes(
+        self, system_volumes: Sequence[DockerVolume], uses_bulkdata_default: bool = True
+    ) -> List[DockerVolume]:
         volumes = list(system_volumes) + list(self.get_extra_volumes())
         # we used to add bulkdata as a default mount - but as part of the
         # effort to deprecate the entire system, we're swapping to an opt-in
         # model so that we can shrink the blast radius of any changes
-        if self.has_bulkdata():
+        if self.config_dict.get(
+            "uses_bulkdata",
+            uses_bulkdata_default,
+        ):
             # bulkdata is mounted RO as the data is produced by another
             # system and we want to ensure that there are no inadvertent
             # changes by misbehaved code
@@ -1038,7 +1040,8 @@ class InstanceConfig:
 
         Defaults to None if not specified in the config.
 
-        :returns: A list of dictionaries specified in the dependencies_dict, None if not specified"""
+        :returns: A list of dictionaries specified in the dependencies_dict, None if not specified
+        """
         dependencies = self.config_dict.get("dependencies")
         if not dependencies:
             return None
@@ -1119,7 +1122,6 @@ def compose(
 
 
 class PaastaColors:
-
     """Collection of static variables and methods to assist in coloring text."""
 
     # ANSI color codes
@@ -2043,12 +2045,12 @@ class SystemPaastaConfigDict(TypedDict, total=False):
     service_auth_token_settings: ProjectedSAVolume
     service_auth_vault_role: str
     always_authenticating_services: List[str]
-    mysql_port_mappings: Dict
     vitess_images: Dict
     superregion_to_region_mapping: Dict
     vitess_tablet_types: List[str]
     vitess_tablet_pool_type_mapping: Dict
     vitess_throttling_config: Dict
+    uses_bulkdata_default: bool
 
 
 def load_system_paasta_config(
@@ -2419,7 +2421,8 @@ class SystemPaastaConfig:
         """Get a format string for the URL to query for haproxy-synapse state. This format string gets two keyword
         arguments, host and port. Defaults to "http://{host:s}:{port:d}/;csv;norefresh".
 
-        :returns: A format string for constructing the URL of haproxy-synapse's status page."""
+        :returns: A format string for constructing the URL of haproxy-synapse's status page.
+        """
         return self.config_dict.get(
             "synapse_haproxy_url_format", DEFAULT_SYNAPSE_HAPROXY_URL_FORMAT
         )
@@ -2434,7 +2437,8 @@ class SystemPaastaConfig:
         """Get a format string that constructs a DNS name pointing at the paasta masters in a cluster. This format
         string gets one parameter: cluster. Defaults to 'paasta-{cluster:s}.yelp'.
 
-        :returns: A format string for constructing the FQDN of the masters in a given cluster."""
+        :returns: A format string for constructing the FQDN of the masters in a given cluster.
+        """
         return self.config_dict.get("cluster_fqdn_format", "paasta-{cluster:s}.yelp")
 
     def get_paasta_status_version(self) -> str:
@@ -2763,9 +2767,6 @@ class SystemPaastaConfig:
     def get_always_authenticating_services(self) -> List[str]:
         return self.config_dict.get("always_authenticating_services", [])
 
-    def get_mysql_port_mappings(self) -> Dict:
-        return self.config_dict.get("mysql_port_mappings", {})
-
     def get_vitess_images(self) -> Dict:
         return self.config_dict.get(
             "vitess_images",
@@ -2809,6 +2810,9 @@ class SystemPaastaConfig:
             },
         )
 
+    def get_uses_bulkdata_default(self) -> bool:
+        return self.config_dict.get("uses_bulkdata_default", True)
+
 
 def _run(
     command: Union[str, List[str]],

{paasta-tools-1.4.21 → paasta-tools-1.4.23}/paasta_tools/vitesscluster_tools.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import sys
 from typing import Any
@@ -15,6 +16,7 @@ from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig
 from paasta_tools.kubernetes_tools import KubernetesDeploymentConfigDict
 from paasta_tools.kubernetes_tools import limit_size_with_hash
 from paasta_tools.kubernetes_tools import sanitised_cr_name
+from paasta_tools.long_running_service_tools import load_service_namespace_config
 from paasta_tools.utils import BranchDictV2
 from paasta_tools.utils import deep_merge_dictionaries
 from paasta_tools.utils import DEFAULT_SOA_DIR
@@ -134,6 +136,7 @@ class GatewayConfigDict(TypedDict, total=False):
     extraLabels: Dict[str, str]
     replicas: int
     resources: Dict[str, Any]
+    annotations: Mapping[str, Any]
 
 
 class CellConfigDict(TypedDict, total=False):
@@ -149,6 +152,7 @@ class VitessDashboardConfigDict(TypedDict, total=False):
     extraLabels: Dict[str, str]
     replicas: int
     resources: Dict[str, Any]
+    annotations: Mapping[str, Any]
 
 
 class VtAdminConfigDict(TypedDict, total=False):
@@ -162,6 +166,7 @@ class VtAdminConfigDict(TypedDict, total=False):
     readOnly: bool
     apiResources: Dict[str, Any]
     webResources: Dict[str, Any]
+    annotations: Mapping[str, Any]
 
 
 class VtTabletDict(TypedDict, total=False):
@@ -182,6 +187,7 @@ class TabletPoolDict(TypedDict, total=False):
     vttablet: VtTabletDict
     externalDatastore: Dict[str, Any]
     dataVolumeClaimTemplate: Dict[str, Any]
+    annotations: Mapping[str, Any]
 
 
 class ShardTemplateDict(TypedDict, total=False):
@@ -229,6 +235,7 @@ def get_cell_config(
     env: List[Union[KVEnvVar, KVEnvVarValueFrom]],
     labels: Dict[str, str],
     node_affinity: dict,
+    annotations: Mapping[str, Any],
 ) -> CellConfigDict:
     """
     get vtgate config
@@ -264,6 +271,7 @@ def get_cell_config(
                 "requests": requests,
                 "limits": requests,
             },
+            annotations=annotations,
         ),
     )
     return config
@@ -276,6 +284,7 @@ def get_vitess_dashboard_config(
     env: List[Union[KVEnvVar, KVEnvVarValueFrom]],
     labels: Dict[str, str],
     node_affinity: dict,
+    annotations: Mapping[str, Any],
 ) -> VitessDashboardConfigDict:
     """
     get vtctld config
@@ -303,6 +312,7 @@ def get_vitess_dashboard_config(
             "requests": requests,
             "limits": requests,
         },
+        annotations=annotations,
     )
     return config
 
@@ -313,6 +323,7 @@ def get_vt_admin_config(
     env: List[Union[KVEnvVar, KVEnvVarValueFrom]],
     labels: Dict[str, str],
     node_affinity: dict,
+    annotations: Mapping[str, Any],
 ) -> VtAdminConfigDict:
     """
     get vtadmin config
@@ -338,6 +349,7 @@ def get_vt_admin_config(
             "requests": requests,
             "limits": requests,
         },
+        annotations=annotations,
     )
     return config
 
@@ -346,7 +358,7 @@ def get_tablet_pool_config(
     cell: str,
     db_name: str,
     keyspace: str,
-    port: str,
+    host: str,
     zk_address: str,
     throttle_query_table: str,
     throttle_metrics_threshold: str,
@@ -356,6 +368,7 @@ def get_tablet_pool_config(
     env: List[Union[KVEnvVar, KVEnvVarValueFrom]],
     labels: Dict[str, str],
     node_affinity: dict,
+    annotations: Mapping[str, Any],
 ) -> TabletPoolDict:
     """
     get vttablet config
@@ -465,8 +478,8 @@ def get_tablet_pool_config(
         },
         externalDatastore={
             "database": db_name,
-            "host": SOURCE_DB_HOST,
-            "port": port,
+            "host": host,
+            "port": 3306,
             "user": "vt_app",
             "credentialsSecret": {
                 "key": "/etc/credentials.yaml",
@@ -478,6 +491,7 @@ def get_tablet_pool_config(
             "resources": {"requests": {"storage": "10Gi"}},
             "storageClassName": "ebs-csi-gp3",
         },
+        annotations=annotations,
     )
     return config
 
@@ -490,6 +504,7 @@ def get_keyspaces_config(
     env: List[Union[KVEnvVar, KVEnvVarValueFrom]],
     labels: Dict[str, str],
     node_affinity: dict,
+    annotations: Mapping[str, Any],
 ) -> List[KeyspaceConfigDict]:
     """
     get vitess keyspace config
@@ -504,19 +519,11 @@ def get_keyspaces_config(
 
         tablet_pools = []
 
-        mysql_port_mappings = load_system_paasta_config().get_mysql_port_mappings()
-
         # get vttablets
         tablet_types = load_system_paasta_config().get_vitess_tablet_types()
         for tablet_type in tablet_types:
-            # We don't have migration or reporting tablets in all clusters
-            if cluster not in mysql_port_mappings:
-                log.error(
-                    f"MySQL Cluster {cluster} not found in system paasta config mysql_port_mappings"
-                )
-            if tablet_type not in mysql_port_mappings[cluster]:
-                continue
-            port = mysql_port_mappings[cluster][tablet_type]
+            ecosystem = region.split("-")[-1]
+            host = f"mysql-{cluster}-{tablet_type}.dre-{ecosystem}"
 
             # We use migration_replication delay for migration tablets and read_replication_delay for everything else
             # Also throttling threshold for refresh and sanitized primaries is set at 30 seconds and everything else at 3 seconds
@@ -546,7 +553,7 @@ def get_keyspaces_config(
                         cell,
                         db_name,
                         keyspace,
-                        port,
+                        host,
                         zk_address,
                         throttle_query_table,
                         throttle_metrics_threshold,
@@ -556,6 +563,7 @@ def get_keyspaces_config(
                         env,
                         labels,
                         node_affinity,
+                        annotations,
                     )
                     for cell in cells
                 ]
@@ -663,6 +671,22 @@ class VitessDeploymentConfig(KubernetesDeploymentConfig):
             labels["yelp.com/owner"] = "dre_mysql"
         return labels
 
+    def get_annotations(self) -> Mapping[str, Any]:
+        # get required annotations to be added to the formatted resource before creating or updating custom resource
+        service_namespace_config = load_service_namespace_config(
+            service=self.service, namespace=self.get_nerve_namespace()
+        )
+        system_paasta_config = load_system_paasta_config()
+        has_routable_ip = self.has_routable_ip(
+            service_namespace_config, system_paasta_config
+        )
+        annotations: Mapping[str, Any] = {
+            "smartstack_registrations": json.dumps(self.get_registrations()),
+            "paasta.yelp.com/routable_ip": has_routable_ip,
+        }
+
+        return annotations
+
     def get_vitess_node_affinity(self) -> dict:
         # Workaround from https://github.com/kubernetes-client/python/issues/390
         api_client = ApiClient()
@@ -706,10 +730,17 @@ class VitessDeploymentConfig(KubernetesDeploymentConfig):
         formatted_env = self.get_env_variables()
         labels = self.get_labels()
         node_affinity = self.get_vitess_node_affinity()
+        annotations = self.get_annotations()
 
         return [
             get_cell_config(
-                cell, region, vtgate_resources, formatted_env, labels, node_affinity
+                cell,
+                region,
+                vtgate_resources,
+                formatted_env,
+                labels,
+                node_affinity,
+                annotations,
             )
             for cell in cells
         ]
@@ -722,9 +753,16 @@ class VitessDeploymentConfig(KubernetesDeploymentConfig):
         formatted_env = self.get_env_variables()
         labels = self.get_labels()
         node_affinity = self.get_vitess_node_affinity()
+        annotations = self.get_annotations()
 
         return get_vitess_dashboard_config(
-            cells, zk_address, vtctld_resources, formatted_env, labels, node_affinity
+            cells,
+            zk_address,
+            vtctld_resources,
+            formatted_env,
+            labels,
+            node_affinity,
+            annotations,
         )
 
     def get_vtadmin(self) -> VtAdminConfigDict:
@@ -734,9 +772,10 @@ class VitessDeploymentConfig(KubernetesDeploymentConfig):
         formatted_env = self.get_env_variables()
         labels = self.get_labels()
         node_affinity = self.get_vitess_node_affinity()
+        annotations = self.get_annotations()
 
         return get_vt_admin_config(
-            cells, vtadmin_resources, formatted_env, labels, node_affinity
+            cells, vtadmin_resources, formatted_env, labels, node_affinity, annotations
         )
 
     def get_keyspaces(self) -> List[KeyspaceConfigDict]:
@@ -748,9 +787,17 @@ class VitessDeploymentConfig(KubernetesDeploymentConfig):
         formatted_env = self.get_env_variables()
         labels = self.get_labels()
         node_affinity = self.get_vitess_node_affinity()
+        annotations = self.get_annotations()
 
         return get_keyspaces_config(
-            cells, keyspaces, zk_address, region, formatted_env, labels, node_affinity
+            cells,
+            keyspaces,
+            zk_address,
+            region,
+            formatted_env,
+            labels,
+            node_affinity,
+            annotations,
         )
 
     def get_update_strategy(self) -> Dict[str, str]:

{paasta-tools-1.4.21 → paasta-tools-1.4.23/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.4.21
+Version: 1.4.23
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp