paasta-tools 1.4.23__tar.gz → 1.4.24__tar.gz

{paasta-tools-1.4.23/paasta_tools.egg-info → paasta-tools-1.4.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.4.23
+Version: 1.4.24
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.4.23 → paasta-tools-1.4.24}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.4.23"
+__version__ = "1.4.24"

{paasta-tools-1.4.23 → paasta-tools-1.4.24}/paasta_tools/cli/cmds/local_run.py

@@ -1209,7 +1209,10 @@ def configure_and_run_docker_container(
     if pull_image:
         docker_pull_image(docker_url)
 
-    for volume in instance_config.get_volumes(system_paasta_config.get_volumes()):
+    for volume in instance_config.get_volumes(
+        system_paasta_config.get_volumes(),
+        system_paasta_config.get_uses_bulkdata_default(),
+    ):
         if os.path.exists(volume["hostPath"]):
             volumes.append(
                 "{}:{}:{}".format(

{paasta-tools-1.4.23 → paasta-tools-1.4.24}/paasta_tools/cli/cmds/spark_run.py

@@ -1189,7 +1189,10 @@ def paasta_spark_run(args: argparse.Namespace) -> int:
     if docker_image_digest is None:
         return 1
 
-    volumes = instance_config.get_volumes(system_paasta_config.get_volumes())
+    volumes = instance_config.get_volumes(
+        system_paasta_config.get_volumes(),
+        system_paasta_config.get_uses_bulkdata_default(),
+    )
     app_base_name = get_spark_app_name(args.cmd or instance_config.get_cmd())
 
     user_spark_opts = _parse_user_spark_args(args.spark_args)

{paasta-tools-1.4.23 → paasta-tools-1.4.24}/paasta_tools/cli/cmds/status.py

@@ -95,14 +95,18 @@ from paasta_tools.utils import load_system_paasta_config
 from paasta_tools.utils import PaastaColors
 from paasta_tools.utils import remove_ansi_escape_sequences
 from paasta_tools.utils import SystemPaastaConfig
-from paasta_tools.vitesscluster_tools import VitessDeploymentConfig
+from paasta_tools.vitesscell_tools import VitessCellConfig
+from paasta_tools.vitesscluster_tools import VitessClusterConfig
+from paasta_tools.vitesskeyspace_tools import VitessKeyspaceConfig
 
 FLINK_STATUS_MAX_THREAD_POOL_WORKERS = 50
 ALLOWED_INSTANCE_CONFIG: Sequence[Type[InstanceConfig]] = [
     FlinkDeploymentConfig,
     FlinkEksDeploymentConfig,
     CassandraClusterDeploymentConfig,
-    VitessDeploymentConfig,
+    VitessClusterConfig,
+    VitessCellConfig,
+    VitessKeyspaceConfig,
     KafkaClusterDeploymentConfig,
     KubernetesDeploymentConfig,
     EksDeploymentConfig,
@@ -115,7 +119,9 @@ DEPLOYMENT_INSTANCE_CONFIG: Sequence[Type[InstanceConfig]] = [
     FlinkDeploymentConfig,
     FlinkEksDeploymentConfig,
     CassandraClusterDeploymentConfig,
-    VitessDeploymentConfig,
+    VitessClusterConfig,
+    VitessCellConfig,
+    VitessKeyspaceConfig,
     KafkaClusterDeploymentConfig,
     KubernetesDeploymentConfig,
     EksDeploymentConfig,
@@ -137,7 +143,9 @@ InstanceStatusWriter = Callable[
 EKS_DEPLOYMENT_CONFIGS = [
     EksDeploymentConfig,
     FlinkEksDeploymentConfig,
-    VitessDeploymentConfig,
+    VitessClusterConfig,
+    VitessCellConfig,
+    VitessKeyspaceConfig,
 ]
 FLINK_DEPLOYMENT_CONFIGS = [FlinkDeploymentConfig, FlinkEksDeploymentConfig]
 

{paasta-tools-1.4.23 → paasta-tools-1.4.24}/paasta_tools/cli/utils.py

@@ -76,7 +76,9 @@ from paasta_tools.utils import PAASTA_K8S_INSTANCE_TYPES
 from paasta_tools.utils import PaastaColors
 from paasta_tools.utils import SystemPaastaConfig
 from paasta_tools.utils import validate_service_instance
-from paasta_tools.vitesscluster_tools import load_vitess_instance_config
+from paasta_tools.vitesscell_tools import load_vitess_cell_instance_config
+from paasta_tools.vitesscluster_tools import load_vitess_cluster_instance_config
+from paasta_tools.vitesskeyspace_tools import load_vitess_keyspace_instance_config
 
 try:
     from vault_tools.paasta_secret import get_client as get_vault_client
@@ -680,7 +682,13 @@ INSTANCE_TYPE_HANDLERS: Mapping[str, InstanceTypeHandler] = defaultdict(
         get_service_instance_list, load_kafkacluster_instance_config
     ),
     vitesscluster=InstanceTypeHandler(
-        get_service_instance_list, load_vitess_instance_config
+        get_service_instance_list, load_vitess_cluster_instance_config
+    ),
+    vitesscell=InstanceTypeHandler(
+        get_service_instance_list, load_vitess_cell_instance_config
+    ),
+    vitesskeyspace=InstanceTypeHandler(
+        get_service_instance_list, load_vitess_keyspace_instance_config
     ),
     nrtsearchservice=InstanceTypeHandler(
         get_service_instance_list, load_nrtsearchservice_instance_config
@@ -713,7 +721,13 @@ LONG_RUNNING_INSTANCE_TYPE_HANDLERS: Mapping[
         get_service_instance_list, load_kafkacluster_instance_config
     ),
     vitesscluster=LongRunningInstanceTypeHandler(
-        get_service_instance_list, load_vitess_instance_config
+        get_service_instance_list, load_vitess_cluster_instance_config
+    ),
+    vitesscell=LongRunningInstanceTypeHandler(
+        get_service_instance_list, load_vitess_cell_instance_config
+    ),
+    vitesskeyspace=LongRunningInstanceTypeHandler(
+        get_service_instance_list, load_vitess_keyspace_instance_config
     ),
     nrtsearchservice=LongRunningInstanceTypeHandler(
         get_service_instance_list, load_nrtsearchservice_instance_config

{paasta-tools-1.4.23 → paasta-tools-1.4.24}/paasta_tools/instance/kubernetes.py

@@ -33,7 +33,9 @@ from paasta_tools import kubernetes_tools
 from paasta_tools import monkrelaycluster_tools
 from paasta_tools import nrtsearchservice_tools
 from paasta_tools import smartstack_tools
+from paasta_tools import vitesscell_tools
 from paasta_tools import vitesscluster_tools
+from paasta_tools import vitesskeyspace_tools
 from paasta_tools.cli.utils import LONG_RUNNING_INSTANCE_TYPE_HANDLERS
 from paasta_tools.instance.hpa_metrics_parser import HPAMetricsDict
 from paasta_tools.instance.hpa_metrics_parser import HPAMetricsParser
@@ -57,6 +59,8 @@ INSTANCE_TYPES_CR = {
     "cassandracluster",
     "kafkacluster",
     "vitesscluster",
+    "vitesscell",
+    "vitesskeyspace",
 }
 INSTANCE_TYPES_K8S = {
     "cassandracluster",
@@ -72,6 +76,8 @@ INSTANCE_TYPE_CR_ID = dict(
     cassandracluster=cassandracluster_tools.cr_id,
     kafkacluster=kafkacluster_tools.cr_id,
     vitesscluster=vitesscluster_tools.cr_id,
+    vitesscell=vitesscell_tools.cr_id,
+    vitesskeyspace=vitesskeyspace_tools.cr_id,
     nrtsearchservice=nrtsearchservice_tools.cr_id,
     nrtsearchserviceeks=nrtsearchservice_tools.cr_id,
     monkrelaycluster=monkrelaycluster_tools.cr_id,

{paasta-tools-1.4.23 → paasta-tools-1.4.24}/paasta_tools/setup_kubernetes_cr.py

@@ -49,13 +49,19 @@ from paasta_tools.utils import get_config_hash
 from paasta_tools.utils import get_git_sha_from_dockerurl
 from paasta_tools.utils import load_all_configs
 from paasta_tools.utils import load_system_paasta_config
-from paasta_tools.vitesscluster_tools import load_vitess_service_instance_configs
+from paasta_tools.vitesscell_tools import load_vitess_cell_instance_configs
+from paasta_tools.vitesscluster_tools import load_vitess_cluster_instance_configs
+from paasta_tools.vitesskeyspace_tools import load_vitess_keyspace_instance_configs
 
 
 log = logging.getLogger(__name__)
 
 
-INSTANCE_TYPE_TO_CONFIG_LOADER = {"vitesscluster": load_vitess_service_instance_configs}
+INSTANCE_TYPE_TO_CONFIG_LOADER = {
+    "vitesscluster": load_vitess_cluster_instance_configs,
+    "vitesscell": load_vitess_cell_instance_configs,
+    "vitesskeyspace": load_vitess_keyspace_instance_configs,
+}
 
 
 class StdoutKubeClient:

{paasta-tools-1.4.23 → paasta-tools-1.4.24}/paasta_tools/utils.py

@@ -137,6 +137,8 @@ INSTANCE_TYPES = (
     "cassandracluster",
     "kafkacluster",
     "vitesscluster",
+    "vitesscell",
+    "vitesskeyspace",
     "monkrelays",
     "nrtsearchservice",
     "nrtsearchserviceeks",
@@ -156,6 +158,8 @@ INSTANCE_TYPE_TO_K8S_NAMESPACE = {
     "cassandracluster": "paasta-cassandraclusters",
     "kafkacluster": "paasta-kafkaclusters",
     "vitesscluster": "paasta-vitessclusters",
+    "vitesscell": "paasta-vitessclusters",
+    "vitesskeyspace": "paasta-vitessclusters",
     "nrtsearchservice": "paasta-nrtsearchservices",
     "nrtsearchserviceeks": "paasta-nrtsearchservices",
 }

paasta-tools-1.4.24/paasta_tools/vitesscell_tools.py

@@ -0,0 +1,289 @@
+import logging
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Mapping
+from typing import Optional
+from typing import TypedDict
+from typing import Union
+
+import service_configuration_lib
+
+from paasta_tools.kubernetes_tools import KubernetesDeploymentConfigDict
+from paasta_tools.kubernetes_tools import sanitised_cr_name
+from paasta_tools.utils import BranchDictV2
+from paasta_tools.utils import deep_merge_dictionaries
+from paasta_tools.utils import DEFAULT_SOA_DIR
+from paasta_tools.utils import load_service_instance_config
+from paasta_tools.utils import load_system_paasta_config
+from paasta_tools.utils import load_v2_deployments_json
+from paasta_tools.vitesscluster_tools import get_formatted_environment_variables
+from paasta_tools.vitesscluster_tools import KVEnvVar
+from paasta_tools.vitesscluster_tools import KVEnvVarValueFrom
+from paasta_tools.vitesscluster_tools import RequestsDict
+from paasta_tools.vitesscluster_tools import ResourceConfigDict
+from paasta_tools.vitesscluster_tools import VitessDeploymentConfig
+from paasta_tools.vitesscluster_tools import VitessDeploymentConfigDict
+
+
+log = logging.getLogger(__name__)
+log.addHandler(logging.NullHandler())
+
+
+KUBERNETES_NAMESPACE = "paasta-vitessclusters"
+
+
+# Global variables
+TOPO_IMPLEMENTATION = "zk2"
+TOPO_GLOBAL_ROOT = "/vitess-paasta/global"
+
+VTGATE_EXTRA_ENV = {
+    "VAULT_ROLEID": {
+        "secretKeyRef": {
+            "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vtgate-approle-roleid",
+            "key": "vault-vtgate-approle-roleid",
+        }
+    },
+    "VAULT_SECRETID": {
+        "secretKeyRef": {
+            "name": "paasta-vitessclusters-secret-vitess-k8s-vault-vtgate-approle-secretid",
+            "key": "vault-vtgate-approle-secretid",
+        }
+    },
+}
+
+
+class GatewayConfigDict(TypedDict, total=False):
+    affinity: Dict[str, Any]
+    extraEnv: List[Union[KVEnvVar, KVEnvVarValueFrom]]
+    extraFlags: Dict[str, str]
+    extraLabels: Dict[str, str]
+    replicas: int
+    resources: Dict[str, Any]
+    annotations: Mapping[str, Any]
+
+
+class VitessCellConfigDict(VitessDeploymentConfigDict, total=False):
+    name: str
+    allCells: List[str]
+    globalLockserver: Dict[str, str]
+    gateway: GatewayConfigDict
+
+
+def get_cell_config(
+    cell: str,
+    images: Dict[str, str],
+    allCells: List[str],
+    global_lock_server: Dict[str, str],
+    region: str,
+    vtgate_resources: ResourceConfigDict,
+    env: List[Union[KVEnvVar, KVEnvVarValueFrom]],
+    labels: Dict[str, str],
+    node_affinity: dict,
+    annotations: Mapping[str, Any],
+) -> VitessCellConfigDict:
+    """
+    get vtgate config
+    """
+    replicas = vtgate_resources.get("replicas")
+    requests = vtgate_resources.get(
+        "requests", RequestsDict(cpu="100m", memory="256Mi")
+    )
+    environment_overrides: Dict[str, Any] = {
+        "VAULT_ADDR": f"https://vault-dre.{region}.yelpcorp.com:8200",
+        "VAULT_CACERT": f"/etc/vault/all_cas/acm-privateca-{region}.crt",
+    }
+    environment_overrides.update(VTGATE_EXTRA_ENV)
+    updated_vtgate_extra_env = (
+        get_formatted_environment_variables(environment_overrides) + env
+    )
+
+    config = VitessCellConfigDict(
+        name=cell,
+        images=images,
+        allCells=allCells,
+        globalLockserver=global_lock_server,
+        gateway=GatewayConfigDict(
+            affinity={"nodeAffinity": node_affinity},
+            extraEnv=updated_vtgate_extra_env,
+            extraFlags={
+                "mysql_auth_server_impl": "vault",
+                "mysql_auth_vault_addr": f"https://vault-dre.{region}.yelpcorp.com:8200",
+                "mysql_auth_vault_path": "secrets/vitess/vt-gate/vttablet_credentials.json",
+                "mysql_auth_vault_tls_ca": f"/etc/vault/all_cas/acm-privateca-{region}.crt",
+                "mysql_auth_vault_ttl": "60s",
+            },
+            extraLabels=labels,
+            replicas=replicas,
+            resources={
+                "requests": requests,
+                "limits": requests,
+            },
+            annotations=annotations,
+        ),
+    )
+    return config
+
+
+class VitessCellInstanceConfigDict(KubernetesDeploymentConfigDict, total=False):
+    cell: str
+    cells: List[str]
+    zk_address: str
+    vtgate_resources: ResourceConfigDict
+    images: Dict[str, str]
+
+
+class VitessCellConfig(VitessDeploymentConfig):
+    config_dict: VitessCellInstanceConfigDict
+
+    config_filename_prefix = "vitesscell"
+
+    def __init__(
+        self,
+        service: str,
+        cluster: str,
+        instance: str,
+        config_dict: VitessCellConfigDict,
+        branch_dict: Optional[BranchDictV2],
+        soa_dir: str = DEFAULT_SOA_DIR,
+    ) -> None:
+        super().__init__(
+            cluster=cluster,  # superregion
+            instance=instance,  # host-1
+            service=service,  # vitess
+            soa_dir=soa_dir,
+            config_dict=config_dict,
+            branch_dict=branch_dict,
+        )
+
+    def get_images(self) -> Dict[str, str]:
+        vitess_images = self.config_dict.get(
+            "images", load_system_paasta_config().get_vitess_images()
+        )
+        return {
+            "vtctld": vitess_images["vtctld_image"],
+            "vtadmin": vitess_images["vtadmin_image"],
+            "vtgate": vitess_images["vtgate_image"],
+            "vttablet": vitess_images["vttablet_image"],
+        }
+
+    def get_global_lock_server(self) -> Dict[str, str]:
+        zk_address = self.config_dict.get("zk_address")
+        return {
+            "implementation": TOPO_IMPLEMENTATION,
+            "address": zk_address,
+            "rootPath": TOPO_GLOBAL_ROOT,
+        }
+
+    def get_vitess_cell_config(self) -> VitessCellConfigDict:
+        cell = self.config_dict.get("cell")
+        all_cells = self.config_dict.get("cells")
+        images = self.get_images()
+        global_lock_server = self.get_global_lock_server()
+        region = self.get_region()
+        vtgate_resources = self.config_dict.get("vtgate_resources")
+
+        formatted_env = self.get_env_variables()
+        labels = self.get_labels()
+        node_affinity = self.get_vitess_node_affinity()
+        annotations = self.get_annotations()
+
+        return get_cell_config(
+            cell,
+            images,
+            all_cells,
+            global_lock_server,
+            region,
+            vtgate_resources,
+            formatted_env,
+            labels,
+            node_affinity,
+            annotations,
+        )
+
+    def validate(
+        self,
+        params: List[str] = [
+            "cpus",
+            "security",
+            "dependencies_reference",
+            "deploy_group",
+        ],
+    ) -> List[str]:
+        # Use InstanceConfig to validate shared config keys like cpus and mem
+        # TODO: add mem back to this list once we fix PAASTA-15582 and
+        # move to using the same units as flink/marathon etc.
+        error_msgs = super().validate(params=params)
+
+        if error_msgs:
+            name = self.get_instance()
+            return [f"{name}: {msg}" for msg in error_msgs]
+        else:
+            return []
+
+
+def load_vitess_cell_instance_config(
+    service: str,
+    instance: str,
+    cluster: str,
+    load_deployments: bool = True,
+    soa_dir: str = DEFAULT_SOA_DIR,
+) -> VitessCellConfig:
+    general_config = service_configuration_lib.read_service_configuration(
+        service, soa_dir=soa_dir
+    )
+    instance_config = load_service_instance_config(
+        service, instance, "vitesscell", cluster, soa_dir=soa_dir
+    )
+    general_config = deep_merge_dictionaries(
+        overrides=instance_config, defaults=general_config
+    )
+
+    branch_dict: Optional[BranchDictV2] = None
+    if load_deployments:
+        deployments_json = load_v2_deployments_json(service, soa_dir=soa_dir)
+        temp_instance_config = VitessCellConfig(
+            service=service,
+            cluster=cluster,
+            instance=instance,
+            config_dict=general_config,
+            branch_dict=None,
+            soa_dir=soa_dir,
+        )
+        branch = temp_instance_config.get_branch()
+        deploy_group = temp_instance_config.get_deploy_group()
+        branch_dict = deployments_json.get_branch_dict(service, branch, deploy_group)
+
+    vitess_cell_config = VitessCellConfig(
+        service=service,
+        cluster=cluster,
+        instance=instance,
+        config_dict=general_config,
+        branch_dict=branch_dict,
+        soa_dir=soa_dir,
+    )
+
+    return vitess_cell_config
+
+
+def load_vitess_cell_instance_configs(
+    service: str,
+    instance: str,
+    cluster: str,
+    soa_dir: str = DEFAULT_SOA_DIR,
+) -> VitessCellConfigDict:
+    vitess_cell_instance_configs = load_vitess_cell_instance_config(
+        service, instance, cluster, soa_dir=soa_dir
+    ).get_vitess_cell_config()
+    return vitess_cell_instance_configs
+
+
+# TODO: read this from CRD in service configs
+def cr_id(service: str, instance: str) -> Mapping[str, str]:
+    return dict(
+        group="planetscale.com",
+        version="v2",
+        namespace=KUBERNETES_NAMESPACE,
+        plural="vitesscells",
+        name=sanitised_cr_name(service, instance),
+    )