paasta-tools 1.4.5__tar.gz → 1.4.7__tar.gz

{paasta-tools-1.4.5/paasta_tools.egg-info → paasta-tools-1.4.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.4.5
+Version: 1.4.7
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.4.5 → paasta-tools-1.4.7}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.4.5"
+__version__ = "1.4.7"

{paasta-tools-1.4.5 → paasta-tools-1.4.7}/paasta_tools/api/api_docs/swagger.json

@@ -953,6 +953,18 @@
                         }
                     },
                     "description": "Nullable CassandraCluster instance status"
+                },
+                "vitesscluster": {
+                    "type": "object",
+                    "properties": {
+                        "status": {
+                            "$ref": "#/definitions/InstanceStatusVitessCluster"
+                        },
+                        "metadata": {
+                            "$ref": "#/definitions/InstanceMetadataVitessCluster"
+                        }
+                    },
+                    "description": "Nullable VitessCluster instance status and metadata"
                 }
             }
         },
@@ -1737,6 +1749,14 @@
             "type": "object",
             "description": "Cassandra instance status"
         },
+        "InstanceStatusVitessCluster": {
+            "type": "object",
+            "description": "VitessCluster instance status"
+        },
+        "InstanceMetadataVitessCluster": {
+            "type": "object",
+            "description": "VitessCluster instance metadata"
+        },
         "InstanceTasks": {
             "type": "array",
             "description": "List of tasks associated with instance",

{paasta-tools-1.4.5 → paasta-tools-1.4.7}/paasta_tools/cli/cmds/logs.py

@@ -40,7 +40,9 @@ from typing import Tuple
 from typing import Type
 from typing import Union
 
+import a_sync
 import isodate
+import nats
 import pytz
 from dateutil import tz
 
@@ -1166,19 +1168,26 @@ class ScribeLogReader(LogReader):
 
 @register_log_reader("vector-logs")
 class VectorLogsReader(LogReader):
+    SUPPORTS_TAILING = True
     SUPPORTS_TIME = True
 
-    def __init__(self, cluster_map: Mapping[str, Any]) -> None:
+    def __init__(
+        self, cluster_map: Mapping[str, Any], nats_endpoint_map: Mapping[str, Any]
+    ) -> None:
         super().__init__()
 
         if S3LogsReader is None:
             raise Exception("yelp_clog package must be available to use S3LogsReader")
 
         self.cluster_map = cluster_map
+        self.nats_endpoint_map = nats_endpoint_map
 
     def get_superregion_for_cluster(self, cluster: str) -> Optional[str]:
         return self.cluster_map.get(cluster, None)
 
+    def get_nats_endpoint_for_cluster(self, cluster: str) -> Optional[str]:
+        return self.nats_endpoint_map.get(cluster, None)
+
     def print_logs_by_time(
         self,
         service,
@@ -1230,6 +1239,48 @@ class VectorLogsReader(LogReader):
         for line in aggregated_logs:
             print_log(line["raw_line"], levels, raw_mode, strip_headers)
 
+    def tail_logs(
+        self,
+        service: str,
+        levels: Sequence[str],
+        components: Iterable[str],
+        clusters: Sequence[str],
+        instances: List[str],
+        pods: Iterable[str] = None,
+        raw_mode: bool = False,
+        strip_headers: bool = False,
+    ) -> None:
+        stream_name = get_log_name_for_service(service, prefix="app_output")
+        endpoint = self.get_nats_endpoint_for_cluster(clusters[0])
+        if not endpoint:
+            raise NotImplementedError(
+                "Tailing logs is not supported in this cluster yet, sorry"
+            )
+
+        async def tail_logs_from_nats() -> None:
+            nc = await nats.connect(f"nats://{endpoint}")
+            sub = await nc.subscribe(stream_name)
+
+            while True:
+                # Wait indefinitely for a new message (no timeout)
+                msg = await sub.next_msg(timeout=None)
+                decoded_data = msg.data.decode("utf-8")
+
+                if paasta_log_line_passes_filter(
+                    decoded_data,
+                    levels,
+                    service,
+                    components,
+                    clusters,
+                    instances,
+                    pods,
+                ):
+                    await a_sync.run(
+                        print_log, decoded_data, levels, raw_mode, strip_headers
+                    )
+
+        a_sync.block(tail_logs_from_nats)
+
 
 def scribe_env_to_locations(scribe_env) -> Mapping[str, Any]:
     """Converts a scribe environment to a dictionary of locations. The

{paasta-tools-1.4.5 → paasta-tools-1.4.7}/paasta_tools/cli/cmds/status.py

@@ -37,6 +37,7 @@ from typing import Optional
 from typing import Sequence
 from typing import Tuple
 from typing import Type
+from typing import TypedDict
 from typing import Union
 
 import a_sync
@@ -94,12 +95,14 @@ from paasta_tools.utils import load_system_paasta_config
 from paasta_tools.utils import PaastaColors
 from paasta_tools.utils import remove_ansi_escape_sequences
 from paasta_tools.utils import SystemPaastaConfig
+from paasta_tools.vitesscluster_tools import VitessDeploymentConfig
 
 FLINK_STATUS_MAX_THREAD_POOL_WORKERS = 50
 ALLOWED_INSTANCE_CONFIG: Sequence[Type[InstanceConfig]] = [
     FlinkDeploymentConfig,
     FlinkEksDeploymentConfig,
     CassandraClusterDeploymentConfig,
+    VitessDeploymentConfig,
     KafkaClusterDeploymentConfig,
     KubernetesDeploymentConfig,
     EksDeploymentConfig,
@@ -112,6 +115,7 @@ DEPLOYMENT_INSTANCE_CONFIG: Sequence[Type[InstanceConfig]] = [
     FlinkDeploymentConfig,
     FlinkEksDeploymentConfig,
     CassandraClusterDeploymentConfig,
+    VitessDeploymentConfig,
     KafkaClusterDeploymentConfig,
     KubernetesDeploymentConfig,
     EksDeploymentConfig,
@@ -130,7 +134,11 @@ InstanceStatusWriter = Callable[
     int,
 ]
 
-EKS_DEPLOYMENT_CONFIGS = [EksDeploymentConfig, FlinkEksDeploymentConfig]
+EKS_DEPLOYMENT_CONFIGS = [
+    EksDeploymentConfig,
+    FlinkEksDeploymentConfig,
+    VitessDeploymentConfig,
+]
 FLINK_DEPLOYMENT_CONFIGS = [FlinkDeploymentConfig, FlinkEksDeploymentConfig]
 
 
@@ -1878,6 +1886,250 @@ def print_kafka_status(
     return 0
 
 
+class EtcdLockServerStatus(TypedDict, total=False):
+    observedGeneration: int
+    available: str
+    clientServiceName: str
+
+
+class LockServerStatus(TypedDict, total=False):
+    etcd: EtcdLockServerStatus
+
+
+class VitessClusterCellStatus(TypedDict, total=False):
+    pendingChanges: str
+    gatewayAvailable: str
+
+
+class VitessClusterKeyspaceStatus(TypedDict, total=False):
+    pendingChanges: str
+    cells: List[str]
+    desiredShards: int
+    shards: int
+    readyShards: int
+    updatedShards: int
+    desiredTablets: int
+    tablets: int
+    readyTablets: int
+    updatedTablets: int
+
+
+class VitessDashboardStatus(TypedDict, total=False):
+    available: str
+    serviceName: str
+
+
+class VTAdminStatus(TypedDict, total=False):
+    available: str
+    serviceName: str
+
+
+class OrphanStatus(TypedDict, total=False):
+    reason: str
+    message: str
+
+
+class VitessClusterStatus(TypedDict, total=False):
+    observedGeneration: int
+    globalLockserver: LockServerStatus
+    gatewayServiceName: str
+    vitessDashboard: VitessDashboardStatus
+    cells: Dict[str, VitessClusterCellStatus]
+    keyspaces: Dict[str, VitessClusterKeyspaceStatus]
+    vtadmin: VTAdminStatus
+    orphanedCells: Dict[str, OrphanStatus]
+    orphanedKeyspaces: Dict[str, OrphanStatus]
+
+
+def print_vitess_status(
+    cluster: str,
+    service: str,
+    instance: str,
+    output: List[str],
+    vitess_status: Mapping[str, Any],
+    verbose: int = 0,
+) -> int:
+    tab = "    "
+    indent = 1
+
+    status: VitessClusterStatus = vitess_status.get("status")
+    if status is None:
+        output.append(
+            PaastaColors.red("indent * tab + Vitess cluster is not available yet")
+        )
+        return 1
+
+    output.append(indent * tab + "Vitess Cluster:")
+    indent += 1
+
+    output.append(
+        indent * tab
+        + "Observed Generation: "
+        + str(status.get("observedGeneration", 0))
+    )
+    output.append(
+        indent * tab + "Gateway Service Name: " + status.get("gatewayServiceName", "")
+    )
+
+    output.append(indent * tab + "Cells:")
+    indent += 1
+    cells: Dict[str, VitessClusterCellStatus] = status.get("cells")
+    if not cells:
+        output.append(
+            indent * tab + "Cells: " + PaastaColors.red("No cell status available")
+        )
+        return 0
+    for cell, cell_status in cells.items():
+        gateway_available: str = cell_status.get("gatewayAvailable")
+        if gateway_available == "True":
+            output.append(
+                indent * tab
+                + f"Cell: {cell} - VTGate: {PaastaColors.green('available')}"
+            )
+        else:
+            output.append(
+                indent * tab
+                + f"Cell: {cell} - VTGate: {PaastaColors.red('unavailable')}"
+            )
+        cell_pending_changes: str = cell_status.get("pendingChanges", None)
+        if cell_pending_changes:
+            output.append(indent * tab + f"  Pending Changes: {cell_pending_changes}")
+    indent -= 1
+
+    output.append(indent * tab + "Vitess Dashboard:")
+    indent += 1
+    vitess_dashboard: VitessDashboardStatus = status.get("vitessDashboard")
+    if not vitess_dashboard:
+        output.append(
+            indent * tab
+            + "Vitess Dashboard: "
+            + PaastaColors.red("No dashboard status available")
+        )
+        return 0
+    vitess_dashboard_available: str = vitess_dashboard.get("available", "")
+    vitess_dashboard_service_name: str = vitess_dashboard.get("serviceName", "")
+    if vitess_dashboard_available == "True":
+        output.append(
+            indent * tab
+            + f"Vitess Dashboard: {vitess_dashboard_service_name} - {PaastaColors.green('available')}"
+        )
+    else:
+        output.append(
+            indent * tab
+            + f"Vitess Dashboard: {vitess_dashboard_service_name} - {PaastaColors.red('unavailable')}"
+        )
+    indent -= 1
+
+    output.append(indent * tab + "VTAdmin:")
+    indent += 1
+    vtadmin: VTAdminStatus = status.get("vtadmin")
+    if not vtadmin:
+        output.append(
+            indent * tab + "VTAdmin: " + PaastaColors.red("No VTAdmin status available")
+        )
+        return 0
+    vtadmin_available: str = vtadmin.get("available", "")
+    vtadmin_service_name: str = vtadmin.get("serviceName", "")
+    if vtadmin_available == "True":
+        output.append(
+            indent * tab
+            + f"VTAdmin: {vtadmin_service_name} - {PaastaColors.green('available')}"
+        )
+    else:
+        output.append(
+            indent * tab
+            + f"VTAdmin: {vtadmin_service_name} - {PaastaColors.red('unavailable')}"
+        )
+    indent -= 1
+
+    output.append(indent * tab + "Keyspaces:")
+    indent += 1
+    keyspaces: Dict[str, VitessClusterKeyspaceStatus] = status.get("keyspaces")
+    if not keyspaces:
+        output.append(
+            indent * tab
+            + "Keyspaces: "
+            + PaastaColors.red("No keyspace status available")
+        )
+        return 0
+    for keyspace, keyspace_status in keyspaces.items():
+        output.append(indent * tab + f"Keyspace: {keyspace}")
+        indent += 1
+        keyspace_pending_changes: str = keyspace_status.get("pendingChanges", None)
+        if keyspace_pending_changes:
+            output.append(
+                indent * tab
+                + f"Keyspace: {keyspace} - Pending Changes: {keyspace_pending_changes}"
+            )
+        keyspace_cells: List[str] = keyspace_status.get("cells", [])
+        output.append(indent * tab + f"  Cells: {', '.join(keyspace_cells)}")
+        desired_shards: int = keyspace_status.get("desiredShards", 0)
+        shards: int = keyspace_status.get("shards", 0)
+        ready_shards: int = keyspace_status.get("readyShards", 0)
+        updated_shards: int = keyspace_status.get("updatedShards", 0)
+        output.append(
+            indent * tab
+            + f"  Shards: {shards} observed, {ready_shards}/{desired_shards} ready, {updated_shards}/{desired_shards} updated"
+        )
+        desired_tablets: int = keyspace_status.get("desiredTablets", 0)
+        tablets: int = keyspace_status.get("tablets", 0)
+        ready_tablets: int = keyspace_status.get("readyTablets", 0)
+        updated_tablets: int = keyspace_status.get("updatedTablets", 0)
+        output.append(
+            indent * tab
+            + f"  Tablets: {tablets} observed, {ready_tablets}/{desired_tablets} ready, {updated_tablets}/{desired_tablets} updated"
+        )
+        indent -= 1
+    indent -= 1
+
+    # This is not needed when not using etcd. We use zk instead
+    global_lockserver: LockServerStatus = status.get("globalLockserver", {})
+    if global_lockserver:
+        output.append(indent * tab + "Global Lockserver:")
+        indent += 1
+        etcd: EtcdLockServerStatus = global_lockserver.get("etcd")
+        if etcd:
+            output.append(indent * tab + "Global Lockserver:")
+            indent += 1
+            observed_generation: int = etcd.get("observedGeneration", 0)
+            available: str = etcd.get("available", "")
+            client_service_name: str = etcd.get("clientServiceName", "")
+            output.append(
+                indent * tab
+                + f"Observed Generation: {observed_generation}, Available: {available}, Client Service Name: {client_service_name}"
+            )
+        indent -= 1
+
+    # Orphaned Cells are not mandatorily seen each time
+    orphaned_cells: Dict[str, OrphanStatus] = status.get("orphanedCells", {})
+    if orphaned_cells:
+        output.append(indent * tab + "Orphaned Cells:")
+        indent += 1
+        for cell, orphan_status in orphaned_cells.items():
+            orphaned_cell_reason: str = orphan_status.get("reason", "")
+            orphaned_cell_message: str = orphan_status.get("message", "")
+            output.append(
+                indent * tab
+                + f"Cell: {cell} - Reason: {orphaned_cell_reason}, Message: {orphaned_cell_message}"
+            )
+        indent -= 1
+
+    # Orphaned Keyspaces are not mandatorily seen each time
+    orphaned_keyspaces: Dict[str, OrphanStatus] = status.get("orphanedKeyspaces", {})
+    if orphaned_keyspaces:
+        output.append(indent * tab + "Orphaned Keyspaces:")
+        indent += 1
+        for keyspace, orphan_status in orphaned_keyspaces.items():
+            orphaned_keyspace_reason: str = orphan_status.get("reason", "")
+            orphaned_keyspace_message: str = orphan_status.get("message", "")
+            output.append(
+                indent * tab
+                + f"Keyspace: {keyspace} - Reason: {orphaned_keyspace_reason}, Message: {orphaned_keyspace_message}"
+            )
+        indent -= 1
+    return 0
+
+
 def report_status_for_cluster(
     service: str,
     cluster: str,
@@ -2273,4 +2525,5 @@ INSTANCE_TYPE_WRITERS: Mapping[str, InstanceStatusWriter] = defaultdict(
     flinkeks=print_flinkeks_status,
     kafkacluster=print_kafka_status,
     cassandracluster=print_cassandra_status,
+    vitesscluster=print_vitess_status,
 )

{paasta-tools-1.4.5 → paasta-tools-1.4.7}/paasta_tools/kubernetes/application/controller_wrappers.py

@@ -15,6 +15,7 @@ from paasta_tools.eks_tools import load_eks_service_config_no_cache
 from paasta_tools.kubernetes_tools import create_deployment
 from paasta_tools.kubernetes_tools import create_pod_disruption_budget
 from paasta_tools.kubernetes_tools import create_stateful_set
+from paasta_tools.kubernetes_tools import ensure_service_account
 from paasta_tools.kubernetes_tools import KubeClient
 from paasta_tools.kubernetes_tools import KubeDeployment
 from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig
@@ -120,6 +121,25 @@ class Application(ABC):
         """
         self.ensure_pod_disruption_budget(kube_client, self.soa_config.get_namespace())
 
+    def update_dependency_api_objects(self, kube_client: KubeClient) -> None:
+        """
+        Update related Kubernetes API objects that should be updated before the main object,
+        such as service accounts.
+        :param kube_client:
+        """
+        self.ensure_service_account(kube_client)
+
+    def ensure_service_account(self, kube_client: KubeClient) -> None:
+        """
+        Ensure that the service account for this application exists
+        :param kube_client:
+        """
+        ensure_service_account(
+            iam_role=self.soa_config.get_iam_role(),
+            namespace=self.soa_config.get_namespace(),
+            kube_client=kube_client,
+        )
+
     def delete_pod_disruption_budget(self, kube_client: KubeClient) -> None:
         try:
             kube_client.policy.delete_namespaced_pod_disruption_budget(

{paasta-tools-1.4.5 → paasta-tools-1.4.7}/paasta_tools/kubernetes_tools.py

@@ -2220,9 +2220,9 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
             annotations["iam.amazonaws.com/role"] = ""
             iam_role = self.get_iam_role()
             if iam_role:
-                pod_spec_kwargs[
-                    "service_account_name"
-                ] = create_or_find_service_account_name(iam_role, self.get_namespace())
+                pod_spec_kwargs["service_account_name"] = get_service_account_name(
+                    iam_role
+                )
                 if fs_group is None:
                     # We need some reasoable default for group id of a process
                     # running inside the container. Seems like most of such
@@ -4050,12 +4050,9 @@ def get_all_limit_ranges(
 _RE_NORMALIZE_IAM_ROLE = re.compile(r"[^0-9a-zA-Z]+")
 
 
-def create_or_find_service_account_name(
+def get_service_account_name(
     iam_role: str,
-    namespace: str,
     k8s_role: Optional[str] = None,
-    kubeconfig_file: Optional[str] = None,
-    dry_run: bool = False,
 ) -> str:
     # the service account is expected to always be prefixed with paasta- as using the actual namespace
     # potentially wastes a lot of characters (e.g., paasta-nrtsearchservices) that could be used for
@@ -4081,12 +4078,17 @@ def create_or_find_service_account_name(
             "Expected at least one of iam_role or k8s_role to be passed in!"
         )
 
-    # if someone is dry-running paasta_setup_tron_namespace or some other tool that
-    # calls this function, we probably don't want to mutate k8s state :)
-    if dry_run:
-        return sa_name
+    return sa_name
+
+
+def ensure_service_account(
+    iam_role: str,
+    namespace: str,
+    kube_client: KubeClient,
+    k8s_role: Optional[str] = None,
+) -> None:
+    sa_name = get_service_account_name(iam_role, k8s_role)
 
-    kube_client = KubeClient(config_file=kubeconfig_file)
     if not any(
         sa.metadata and sa.metadata.name == sa_name
         for sa in get_all_service_accounts(kube_client, namespace)
@@ -4135,8 +4137,6 @@ def create_or_find_service_account_name(
                 namespace=namespace, body=role_binding
             )
 
-    return sa_name
-
 
 def mode_to_int(mode: Optional[Union[str, int]]) -> Optional[int]:
     if mode is not None:

{paasta-tools-1.4.5 → paasta-tools-1.4.7}/paasta_tools/paastaapi/model/instance_status.py

@@ -38,6 +38,7 @@ def lazy_import():
     from paasta_tools.paastaapi.model.instance_status_kubernetes import InstanceStatusKubernetes
     from paasta_tools.paastaapi.model.instance_status_kubernetes_v2 import InstanceStatusKubernetesV2
     from paasta_tools.paastaapi.model.instance_status_tron import InstanceStatusTron
+    from paasta_tools.paastaapi.model.instance_status_vitesscluster import InstanceStatusVitesscluster
     globals()['InstanceStatusAdhoc'] = InstanceStatusAdhoc
     globals()['InstanceStatusCassandracluster'] = InstanceStatusCassandracluster
     globals()['InstanceStatusFlink'] = InstanceStatusFlink
@@ -45,6 +46,7 @@ def lazy_import():
     globals()['InstanceStatusKubernetes'] = InstanceStatusKubernetes
     globals()['InstanceStatusKubernetesV2'] = InstanceStatusKubernetesV2
     globals()['InstanceStatusTron'] = InstanceStatusTron
+    globals()['InstanceStatusVitesscluster'] = InstanceStatusVitesscluster
 
 
 class InstanceStatus(ModelNormal):
@@ -100,6 +102,7 @@ class InstanceStatus(ModelNormal):
             'version': (str,),  # noqa: E501
             'instance': (str,),  # noqa: E501
             'cassandracluster': (InstanceStatusCassandracluster,),  # noqa: E501
+            'vitesscluster': (InstanceStatusVitesscluster,),  # noqa: E501
             'kafkacluster': (InstanceStatusKafkacluster,),  # noqa: E501
             'kubernetes': (InstanceStatusKubernetes,),  # noqa: E501
             'kubernetes_v2': (InstanceStatusKubernetesV2,),  # noqa: E501
@@ -120,6 +123,7 @@ class InstanceStatus(ModelNormal):
         'version': 'version',  # noqa: E501
         'instance': 'instance',  # noqa: E501
         'cassandracluster': 'cassandracluster',  # noqa: E501
+        'vitesscluster': 'vitesscluster',  # noqa: E501
         'kafkacluster': 'kafkacluster',  # noqa: E501
         'kubernetes': 'kubernetes',  # noqa: E501
         'kubernetes_v2': 'kubernetes_v2',  # noqa: E501
@@ -180,6 +184,7 @@ class InstanceStatus(ModelNormal):
             version (str): Deployment Version of a service. [optional]  # noqa: E501
             instance (str): Instance name. [optional]  # noqa: E501
             cassandracluster (InstanceStatusCassandracluster): [optional]  # noqa: E501
+            vitesscluster (InstanceStatusVitesscluster): [optional]  # noqa: E501
             kafkacluster (InstanceStatusKafkacluster): [optional]  # noqa: E501
             kubernetes (InstanceStatusKubernetes): [optional]  # noqa: E501
             kubernetes_v2 (InstanceStatusKubernetesV2): [optional]  # noqa: E501