paasta-tools 1.23.2__tar.gz → 1.24.0__tar.gz

{paasta_tools-1.23.2/paasta_tools.egg-info → paasta_tools-1.24.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: paasta-tools
-Version: 1.23.2
+Version: 1.24.0
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com
@@ -22,6 +22,7 @@ Requires-Dist: cookiecutter>=1.4.0
 Requires-Dist: croniter
 Requires-Dist: docker
 Requires-Dist: dulwich>=0.17.3
+Requires-Dist: environment-tools
 Requires-Dist: ephemeral-port-reserve>=1.0.1
 Requires-Dist: graphviz
 Requires-Dist: grpcio

{paasta_tools-1.23.2 → paasta_tools-1.24.0}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.23.2"
+__version__ = "1.24.0"

{paasta_tools-1.23.2 → paasta_tools-1.24.0}/paasta_tools/cli/cmds/status.py

@@ -63,13 +63,16 @@ from paasta_tools.flink_tools import FlinkDeploymentConfig
 from paasta_tools.flink_tools import get_flink_config_from_paasta_api_client
 from paasta_tools.flink_tools import get_flink_jobs_from_paasta_api_client
 from paasta_tools.flink_tools import get_flink_overview_from_paasta_api_client
+from paasta_tools.flink_tools import load_flink_instance_config
 from paasta_tools.flinkeks_tools import FlinkEksDeploymentConfig
+from paasta_tools.flinkeks_tools import load_flinkeks_instance_config
 from paasta_tools.kafkacluster_tools import KafkaClusterDeploymentConfig
 from paasta_tools.kubernetes_tools import format_pod_event_messages
 from paasta_tools.kubernetes_tools import format_tail_lines_for_kubernetes_pod
 from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig
 from paasta_tools.kubernetes_tools import KubernetesDeployStatus
 from paasta_tools.kubernetes_tools import paasta_prefixed
+from paasta_tools.monitoring_tools import get_runbook
 from paasta_tools.monitoring_tools import get_team
 from paasta_tools.monitoring_tools import list_teams
 from paasta_tools.paasta_service_config_loader import PaastaServiceConfigLoader
@@ -763,12 +766,20 @@ def append_pod_status(pod_status, output: List[str]):
     output.extend([f"      {line}" for line in pods_table])
 
 
+OUTPUT_HORIZONTAL_RULE = (
+    "=================================================================="
+)
+
+
 def _print_flink_status_from_job_manager(
     service: str,
     instance: str,
+    cluster: str,
     output: List[str],
     flink: Mapping[str, Any],
     client: PaastaOApiClient,
+    system_paasta_config: "SystemPaastaConfig",
+    flink_instance_config: FlinkDeploymentConfig,
     verbose: int,
 ) -> int:
     status = flink.get("status")
@@ -776,6 +787,7 @@ def _print_flink_status_from_job_manager(
         output.append(PaastaColors.red("    Flink cluster is not available yet"))
         return 1
 
+    # Print Flink Config SHA
     # Since metadata should be available no matter the state, we show it first. If this errors out
     # then we cannot really do much to recover, because cluster is not in usable state anyway
     metadata = flink.get("metadata")
@@ -785,9 +797,32 @@ def _print_flink_status_from_job_manager(
         raise ValueError(f"expected config sha on Flink, but received {metadata}")
     if config_sha.startswith("config"):
         config_sha = config_sha[6:]
-
     output.append(f"    Config SHA: {config_sha}")
 
+    if verbose:
+        # Print Flink repo links
+        output.append(f"    Repo(git): https://github.yelpcorp.com/services/{service}")
+        output.append(
+            f"    Repo(sourcegraph): https://sourcegraph.yelpcorp.com/services/{service}"
+        )
+
+        # Print Flink Pool information
+        flink_pool = flink_instance_config.get_pool()
+        output.append(f"    Flink Pool: {flink_pool}")
+
+        # Print ownership information
+        flink_monitoring_team = flink_instance_config.get_team() or get_team(
+            overrides={}, service=service, soa_dir=DEFAULT_SOA_DIR
+        )
+        output.append(f"    Owner: {flink_monitoring_team}")
+
+        # Print rb information
+        flink_rb_for_instance = flink_instance_config.get_runbook() or get_runbook(
+            overrides={}, service=service, soa_dir=DEFAULT_SOA_DIR
+        )
+        output.append(f"    Flink Runbook: {flink_rb_for_instance}")
+
+    # Print Flink Version
     if status["state"] == "running":
         try:
             flink_config = get_flink_config_from_paasta_api_client(
@@ -805,17 +840,67 @@ def _print_flink_status_from_job_manager(
         else:
             output.append(f"    Flink version: {flink_config.flink_version}")
 
+        # Print Flink Dashboard URL
         # Annotation "flink.yelp.com/dashboard_url" is populated by flink-operator
         dashboard_url = metadata["annotations"].get("flink.yelp.com/dashboard_url")
         output.append(f"    URL: {dashboard_url}/")
 
+    if verbose:
+        # Print Flink config link resources
+        ecosystem = system_paasta_config.get_ecosystem_for_cluster(cluster)
+        output.append(
+            f"    Yelpsoa configs: https://github.yelpcorp.com/sysgit/yelpsoa-configs/tree/master/{service}"
+        )
+        output.append(
+            f"    Srv configs: https://github.yelpcorp.com/sysgit/srv-configs/tree/master/ecosystem/{ecosystem}/{service}"
+        )
+
+        output.append(f"{OUTPUT_HORIZONTAL_RULE}")
+
+        # Print Flink Log Commands
+        output.append(f"    Flink Log Commands:")
+        output.append(
+            f"      Service:     paasta logs -a 1h -c {cluster} -s {service} -i {instance}"
+        )
+        output.append(
+            f"      Taskmanager: paasta logs -a 1h -c {cluster} -s {service} -i {instance}.TASKMANAGER"
+        )
+        output.append(
+            f"      Jobmanager:  paasta logs -a 1h -c {cluster} -s {service} -i {instance}.JOBMANAGER"
+        )
+        output.append(
+            f"      Supervisor:  paasta logs -a 1h -c {cluster} -s {service} -i {instance}.SUPERVISOR"
+        )
+
+        output.append(f"{OUTPUT_HORIZONTAL_RULE}")
+
+        # Print Flink Metrics Links
+        output.append(f"    Flink Monitoring:")
+        output.append(
+            f"      Job Metrics: https://grafana.yelpcorp.com/d/flink-metrics/flink-job-metrics?orgId=1&var-datasource=Prometheus-flink&var-region=uswest2-{ecosystem}&var-service={service}&var-instance={instance}&var-job=All&from=now-24h&to=now"
+        )
+        output.append(
+            f"      Container Metrics: https://grafana.yelpcorp.com/d/flink-container-metrics/flink-container-metrics?orgId=1&var-datasource=Prometheus-flink&var-region=uswest2-{ecosystem}&var-service={service}&var-instance={instance}&from=now-24h&to=now"
+        )
+        output.append(
+            f"      JVM Metrics: https://grafana.yelpcorp.com/d/flink-jvm-metrics/flink-jvm-metrics?orgId=1&var-datasource=Prometheus-flink&var-region=uswest2-{ecosystem}&var-service={service}&var-instance={instance}&from=now-24h&to=now"
+        )
+
+        # Print Flink Costs Link
+        output.append(
+            f"      Flink Cost: https://splunk.yelpcorp.com/en-US/app/yelp_computeinfra/paasta_service_utilization?form.service={service}&form.field1.earliest=-30d%40d&form.field1.latest=now&form.instance={instance}&form.cluster={cluster}"
+        )
+
+        output.append(f"{OUTPUT_HORIZONTAL_RULE}")
+
+    # Print Flink Cluster State
     color = PaastaColors.green if status["state"] == "running" else PaastaColors.yellow
     output.append(f"    State: {color(status['state'].title())}")
 
+    # Print Flink Cluster Pod Info
     pod_running_count = pod_evicted_count = pod_other_count = 0
     # default for evicted in case where pod status is not available
     evicted = f"{pod_evicted_count}"
-
     for pod in status["pod_status"]:
         if pod["phase"] == "Running":
             pod_running_count += 1
@@ -987,8 +1072,22 @@ def print_flink_status(
         )
         return 1
 
+    flink_instance_config = load_flink_instance_config(
+        service=service,
+        instance=instance,
+        cluster=cluster,
+    )
+
     return _print_flink_status_from_job_manager(
-        service, instance, output, flink, client, verbose
+        service,
+        instance,
+        cluster,
+        output,
+        flink,
+        client,
+        system_paasta_config,
+        flink_instance_config,
+        verbose,
     )
 
 
@@ -1013,9 +1112,21 @@ def print_flinkeks_status(
             )
         )
         return 1
-
+    flink_eks_instance_config = load_flinkeks_instance_config(
+        service=service,
+        instance=instance,
+        cluster=cluster,
+    )
     return _print_flink_status_from_job_manager(
-        service, instance, output, flink, client, verbose
+        service,
+        instance,
+        cluster,
+        output,
+        flink,
+        client,
+        system_paasta_config,
+        flink_eks_instance_config,
+        verbose,
     )
 
 

{paasta_tools-1.23.2 → paasta_tools-1.24.0}/paasta_tools/flink_tools.py

@@ -62,6 +62,7 @@ class TaskManagerConfig(TypedDict, total=False):
 
 class FlinkDeploymentConfigDict(LongRunningServiceConfigDict, total=False):
     taskmanager: TaskManagerConfig
+    spot: bool
 
 
 class FlinkDeploymentConfig(LongRunningServiceConfig):
@@ -111,6 +112,23 @@ class FlinkDeploymentConfig(LongRunningServiceConfig):
     def get_replication_crit_percentage(self) -> int:
         return self.config_dict.get("replication_threshold", 100)
 
+    def get_pool(self) -> Optional[str]:
+        """
+        Parses flink_pool from a specific Flink Deployment instance's configuration data, using key 'spot'.
+
+        Args:
+            flink_deployment_config_data: The FlinkDeploymentConfig for a specific Flink yelpsoa instance
+
+        Returns:
+            The flink pool string.
+        """
+        spot_config = self.config_dict.get("spot", None)
+        if spot_config is False:
+            return "flink"
+        else:
+            # if not set or True, Flink instance defaults to use flink-spot pool
+            return "flink-spot"
+
 
 def load_flink_instance_config(
     service: str,

{paasta_tools-1.23.2 → paasta_tools-1.24.0}/paasta_tools/utils.py

@@ -76,6 +76,7 @@ import requests_cache
 import service_configuration_lib
 from docker import APIClient
 from docker.utils import kwargs_from_env
+from environment_tools.type_utils import convert_location_type
 from kazoo.client import KazooClient
 from mypy_extensions import TypedDict
 from service_configuration_lib import read_extra_service_information
@@ -490,6 +491,9 @@ class InstanceConfig:
     def get_team(self) -> str:
         return self.config_dict.get("monitoring", {}).get("team", None)
 
+    def get_runbook(self) -> str:
+        return self.config_dict.get("monitoring", {}).get("runbook", None)
+
     def get_mem(self) -> float:
         """Gets the memory required from the service's configuration.
 
@@ -2802,6 +2806,47 @@ class SystemPaastaConfig:
     def get_remote_run_duration_limit(self, default: int) -> int:
         return self.config_dict.get("remote_run_duration_limit", default)
 
+    def get_ecosystem_for_cluster(self, cluster: str) -> Optional[str]:
+        """
+        Convert a Kubernetes cluster's region information to an ecosystem name.
+
+        This function extracts the 'yelp_region' from the cluster data and
+        converts it to an ecosystem identifier (e.g., 'uswest2-devc' → 'devc').
+
+        Convert the yelp_region to ecosystem
+        See y/habitat for what these mean
+        Example
+         convert_location_type(
+             location="uswest2-devc",
+             source_type="region",
+             desired_type="ecosystem",
+         )
+        Output: devc
+
+        NOTE: kube_clusters_data.get(cluster) returns a string like uswest2-devc
+        which loosely looks like an aws region - but while you can
+        go from yelp region -> aws region, the reverse is not
+        true without additional data
+        """
+        kube_clusters_data = self.get_kube_clusters()
+        cluster_info = kube_clusters_data.get(cluster)
+        if cluster_info is not None:
+            yelp_region = cluster_info.get("yelp_region", None)
+        else:
+            # NOTE: this should never happen unless our kube metadata generator is broken
+            return None
+
+        result = convert_location_type(
+            location=yelp_region,
+            source_type="region",
+            desired_type="ecosystem",
+        )
+        if result:
+            return result[0]
+        else:
+            # NOTE: this should never happen unless we've gotten bad data
+            return None
+
 
 def _run(
     command: Union[str, List[str]],

{paasta_tools-1.23.2 → paasta_tools-1.24.0/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: paasta-tools
-Version: 1.23.2
+Version: 1.24.0
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com
@@ -22,6 +22,7 @@ Requires-Dist: cookiecutter>=1.4.0
 Requires-Dist: croniter
 Requires-Dist: docker
 Requires-Dist: dulwich>=0.17.3
+Requires-Dist: environment-tools
 Requires-Dist: ephemeral-port-reserve>=1.0.1
 Requires-Dist: graphviz
 Requires-Dist: grpcio

{paasta_tools-1.23.2 → paasta_tools-1.24.0}/paasta_tools.egg-info/requires.txt

@@ -13,6 +13,7 @@ cookiecutter>=1.4.0
 croniter
 docker
 dulwich>=0.17.3
+environment-tools
 ephemeral-port-reserve>=1.0.1
 graphviz
 grpcio

{paasta_tools-1.23.2 → paasta_tools-1.24.0}/requirements-minimal.txt

@@ -13,6 +13,7 @@ cookiecutter >= 1.4.0
 croniter
 docker
 dulwich >= 0.17.3
+environment-tools
 ephemeral-port-reserve >= 1.0.1
 graphviz
 grpcio

{paasta_tools-1.23.2 → paasta_tools-1.24.0}/tests/test_flink_tools.py

@@ -14,6 +14,8 @@
 import mock
 
 import paasta_tools.flink_tools as flink_tools
+from paasta_tools.flink_tools import FlinkDeploymentConfig
+from paasta_tools.flink_tools import FlinkDeploymentConfigDict
 
 
 def test_get_flink_ingress_url_root():
@@ -244,3 +246,76 @@ def test_get_flink_jobmanager_overview():
             "flink-version": "1.6.4",
             "flink-commit": "6241481",
         }
+
+
+class TestGetFlinkPoolFromFlinkDeploymentConfig:
+    def test_explicit_spot_false(self):
+        # When spot is explicitly set to False, should return "flink"
+        config_dict = FlinkDeploymentConfigDict({"spot": False})
+        flink_deployment_config = FlinkDeploymentConfig(
+            service="test_service",
+            cluster="test_cluster",
+            instance="test_instance",
+            config_dict=config_dict,
+            branch_dict=None,
+        )
+        assert flink_deployment_config.get_pool() == "flink"
+
+    def test_explicit_spot_true(self):
+        # When spot is explicitly set to True, should return "flink-spot"
+        config_dict = FlinkDeploymentConfigDict({"spot": True})
+        flink_deployment_config = FlinkDeploymentConfig(
+            service="test_service",
+            cluster="test_cluster",
+            instance="test_instance",
+            config_dict=config_dict,
+            branch_dict=None,
+        )
+        assert flink_deployment_config.get_pool() == "flink-spot"
+
+    def test_spot_not_set(self):
+        # When spot is not set, should default to "flink-spot"
+        config_dict = FlinkDeploymentConfigDict({"some_other_key": "value"})
+        flink_deployment_config = FlinkDeploymentConfig(
+            service="test_service",
+            cluster="test_cluster",
+            instance="test_instance",
+            config_dict=config_dict,
+            branch_dict=None,
+        )
+        assert flink_deployment_config.get_pool() == "flink-spot"
+
+    def test_empty_config_dict(self):
+        # When config_dict is empty (but not None), should default to "flink-spot"
+        config_dict = FlinkDeploymentConfigDict({})
+        flink_deployment_config = FlinkDeploymentConfig(
+            service="test_service",
+            cluster="test_cluster",
+            instance="test_instance",
+            config_dict=config_dict,
+            branch_dict=None,
+        )
+        assert flink_deployment_config.get_pool() == "flink-spot"
+
+    def test_non_bool_spot_value(self):
+        # When spot has a non-boolean value like a string, it should treat it as non-False
+        config_dict = FlinkDeploymentConfigDict({"spot": "some_string"})
+        flink_deployment_config = FlinkDeploymentConfig(
+            service="test_service",
+            cluster="test_cluster",
+            instance="test_instance",
+            config_dict=config_dict,
+            branch_dict=None,
+        )
+        assert flink_deployment_config.get_pool() == "flink-spot"
+
+        # Test with a numeric value
+        config_dict = FlinkDeploymentConfigDict({"spot": 0})
+        flink_deployment_config = FlinkDeploymentConfig(
+            service="test_service",
+            cluster="test_cluster",
+            instance="test_instance",
+            config_dict=config_dict,
+            branch_dict=None,
+        )
+        assert flink_deployment_config.get_pool() == "flink-spot"

{paasta_tools-1.23.2 → paasta_tools-1.24.0}/tests/test_utils.py

@@ -591,6 +591,38 @@ def test_SystemPaastaConfig_get_cluster_fqdn_format():
     assert actual == expected
 
 
+def test_SystemPaastaConfig_get_ecosystem_for_cluster():
+    with mock.patch(
+        "paasta_tools.utils.convert_location_type", autospec=True
+    ) as mock_convert_location_type:
+        # Mock convert_location_type to return the expected ecosystem
+        mock_convert_location_type.return_value = ["devc"]
+        # Create a mock SystemPaastaConfig with predefined cluster data
+        fake_system_paasta_config = SystemPaastaConfig(
+            SystemPaastaConfigDict(
+                {
+                    "kube_clusters": {
+                        "fake-cluster": {
+                            "server": "https://k8s.test.paasta:1234",
+                            "certificate-authority-data": "test-cert",
+                            "aws_region": "us-west-2",
+                            "aws_account": "test",
+                            "yelp_region": "uswest2-devc",
+                            "aws_account_id": "123456789012",
+                        }
+                    }
+                }
+            ),
+            "/fake/path",
+        )
+        # Test with existing cluster
+        ecosystem = fake_system_paasta_config.get_ecosystem_for_cluster("fake-cluster")
+        assert ecosystem == "devc"
+        mock_convert_location_type.assert_called_once_with(
+            location="uswest2-devc", source_type="region", desired_type="ecosystem"
+        )
+
+
 @pytest.mark.parametrize(
     "config,expected_git,expected_primary",
     [
@@ -1415,6 +1447,19 @@ class TestInstanceConfig:
             == fake_info
         )
 
+    def test_get_monitoring_runbook(self):
+        fake_info: utils.MonitoringDict = {"runbook": "fake_runbook"}
+        assert (
+            utils.InstanceConfig(
+                service="",
+                cluster="",
+                instance="",
+                config_dict={"monitoring": fake_info},
+                branch_dict=None,
+            ).get_runbook()
+            == "fake_runbook"
+        )
+
     def test_get_cpus_in_config(self):
         fake_conf = utils.InstanceConfig(
             service="",