paasta-tools 1.30.9__py3-none-any.whl → 1.35.8__py3-none-any.whl

paasta_tools/cli/cmds/validate.py

@@ -22,6 +22,7 @@ from datetime import datetime
 from functools import lru_cache
 from functools import partial
 from glob import glob
+from pathlib import Path
 from typing import Any
 from typing import Callable
 from typing import cast
@@ -34,6 +35,9 @@ from typing import Union
 
 import pytz
 from croniter import croniter
+from environment_tools.type_utils import available_location_types
+from environment_tools.type_utils import compare_types
+from environment_tools.type_utils import convert_location_type
 from jsonschema import Draft4Validator
 from jsonschema import exceptions
 from jsonschema import FormatChecker
@@ -64,6 +68,7 @@ from paasta_tools.long_running_service_tools import METRICS_PROVIDER_PISCINA
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_PROMQL
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_UWSGI
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_UWSGI_V2
+from paasta_tools.long_running_service_tools import METRICS_PROVIDER_WORKER_LOAD
 from paasta_tools.secret_tools import get_secret_name_from_ref
 from paasta_tools.secret_tools import is_secret_ref
 from paasta_tools.secret_tools import is_shared_secret
@@ -120,6 +125,7 @@ SCHEMA_TYPES = {
     "rollback",  # automatic rollbacks during deployments
     "tron",  # batch workloads
     "eks",  # eks workloads
+    "smartstack",  # mesh configs
     "autotuned_defaults/kubernetes",
     "autotuned_defaults/cassandracluster",
 }
@@ -163,9 +169,33 @@ INVALID_AUTOSCALING_FIELDS = {
         "desired_active_requests_per_replica",
         "prometheus-adapter-config",
     },
+    METRICS_PROVIDER_WORKER_LOAD: {
+        "desired_active_requests_per_replica",
+        "prometheus-adapter-config",
+    },
     METRICS_PROVIDER_PROMQL: {"desired_active_requests_per_replica"},
 }
 
+# Listener names in Envoy cannot exceed 128 characters and use the
+# following format:
+#    service_name.namespace.listener
+# This naming scheme leaves 128 - 10 = 118 characters
+# for the Smartstack service name and namespace.
+# See COREBACK-6303 for more context.
+MAX_ENVOY_NAME_LEN = 118
+
+# Socket names cannot exceed 108 characters, and the longest socket
+# name generated by HAProxy uses the following format:
+#    /var/run/synapse/sockets/service_name.namespace.LONGPID.sock.tmp
+# This naming scheme leaves 108 - 43 = 65 characters combined for the
+# Smartstack service name and namespace. We leave a generous buffer
+# to arrive at a maximum name length of 55, in case e.g. the .sock
+# suffix is renamed to a longer name for certain sockets.
+# See SMTSTK-204 for more context.
+# NOTE: the above is mostly still true - but the path we use is now /var/run/envoy/sockets/...
+# so we may want to adjust this a tad in the future ;)
+MAX_SMARTSTACK_NAME_LEN = 55
+
 
 class ConditionConfig(TypedDict, total=False):
     """
@@ -339,11 +369,9 @@ def get_config_file_dict(file_path: str, use_ruamel: bool = False) -> Dict[Any,
                 # sets disk: 100 -> an instance uses that template and overwrites
                 # it with disk: 1000)
                 ruamel_loader.allow_duplicate_keys = True
-                # we want to actually expand out all anchors so that we still get
-                # comments from the original block
-                ruamel_loader.Constructor.flatten_mapping = (
-                    SafeConstructor.flatten_mapping
-                )
+                # Note: we do NOT use flatten_mapping here because it breaks simple
+                # override patterns. For nested merge patterns, we load a flattened
+                # version separately for comment detection only.
                 return ruamel_loader.load(config_file)
             else:
                 return yaml.safe_load(config_file)
@@ -430,6 +458,8 @@ def add_subparser(subparsers):
         "--service",
         required=False,
         help="Service that you want to validate. Like 'example_service'.",
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     validate_parser.add_argument(
         "-v",
@@ -607,7 +637,22 @@ def validate_unique_instance_names(service_path):
     return check_passed
 
 
-def _get_comments_for_key(data: CommentedMap, key: Any) -> Optional[str]:
+def _get_config_flattened(file_path: str) -> CommentedMap:
+    """Load config with flatten_mapping enabled (for nested merge pattern comment detection)"""
+    config_file = get_file_contents(file_path)
+    ruamel_loader = YAML(typ="rt")
+    ruamel_loader.allow_duplicate_keys = True
+    ruamel_loader.Constructor.flatten_mapping = SafeConstructor.flatten_mapping
+    return ruamel_loader.load(config_file)
+
+
+def _get_comments_for_key(
+    data: CommentedMap,
+    key: Any,
+    full_config: Optional[Dict[Any, Any]] = None,
+    key_value: Any = None,
+    full_config_flattened: Optional[Dict[Any, Any]] = None,
+) -> Optional[str]:
     # this is a little weird, but ruamel is returning a list that looks like:
     # [None, None, CommentToken(...), None] for some reason instead of just a
     # single string
@@ -624,6 +669,31 @@ def _get_comments_for_key(data: CommentedMap, key: Any) -> Optional[str]:
 
     raw_comments = [*_flatten_comments(data.ca.items.get(key, []))]
     if not raw_comments:
+        # If we didn't find a comment in the instance itself, check if this key
+        # might be inherited from a template. Look for ANY other instance/template
+        # in the config that has the same key with the same value and a comment.
+        if full_config is not None and key_value is not None:
+            for config_key, config_value in full_config.items():
+                if isinstance(config_value, CommentedMap):
+                    if config_value.get(key) == key_value:
+                        other_comments = [
+                            *_flatten_comments(config_value.ca.items.get(key, []))
+                        ]
+                        if other_comments:
+                            return "".join(other_comments)
+
+        # If still not found and we have a flattened config, check there
+        # (flattened config is needed for nested merges)
+        if full_config_flattened is not None:
+            for config_key, config_value in full_config_flattened.items():
+                if isinstance(config_value, CommentedMap):
+                    if config_value.get(key) == key_value:
+                        flattened_comments = [
+                            *_flatten_comments(config_value.ca.items.get(key, []))
+                        ]
+                        if flattened_comments:
+                            return "".join(flattened_comments)
+
         # return None so that we don't return an empty string below if there really aren't
         # any comments
         return None
@@ -772,12 +842,13 @@ def validate_autoscaling_configs(service_path: str) -> bool:
 
                             # we need access to the comments, so we need to read the config with ruamel to be able
                             # to actually get them in a "nice" automated fashion
+                            config_file_path = os.path.join(
+                                soa_dir,
+                                service,
+                                f"{instance_config.get_instance_type()}-{cluster}.yaml",
+                            )
                             config = get_config_file_dict(
-                                os.path.join(
-                                    soa_dir,
-                                    service,
-                                    f"{instance_config.get_instance_type()}-{cluster}.yaml",
-                                ),
+                                config_file_path,
                                 use_ruamel=True,
                             )
                             if config[instance].get("cpus") is None:
@@ -792,8 +863,15 @@ def validate_autoscaling_configs(service_path: str) -> bool:
                                 # cpu autoscaled, but using autotuned values - can skip
                                 continue
 
+                            # Load flattened config for comment detection (handles nested merges)
+                            config_flattened = _get_config_flattened(config_file_path)
+
                             cpu_comment = _get_comments_for_key(
-                                data=config[instance], key="cpus"
+                                data=config[instance],
+                                key="cpus",
+                                full_config=config,
+                                key_value=config[instance].get("cpus"),
+                                full_config_flattened=config_flattened,
                             )
                             # we could probably have a separate error message if there's a comment that doesn't match
                             # the ack pattern, but that seems like overkill - especially for something that could cause
@@ -935,12 +1013,13 @@ def validate_cpu_burst(service_path: str) -> bool:
             if is_k8s_service and not should_skip_cpu_burst_validation:
                 # we need access to the comments, so we need to read the config with ruamel to be able
                 # to actually get them in a "nice" automated fashion
+                config_file_path = os.path.join(
+                    soa_dir,
+                    service,
+                    f"{instance_config.get_instance_type()}-{cluster}.yaml",
+                )
                 config = get_config_file_dict(
-                    os.path.join(
-                        soa_dir,
-                        service,
-                        f"{instance_config.get_instance_type()}-{cluster}.yaml",
-                    ),
+                    config_file_path,
                     use_ruamel=True,
                 )
 
@@ -951,8 +1030,15 @@ def validate_cpu_burst(service_path: str) -> bool:
                     # under the threshold - can also skip
                     continue
 
+                # Load flattened config for comment detection (handles nested merges)
+                config_flattened = _get_config_flattened(config_file_path)
+
                 burst_comment = _get_comments_for_key(
-                    data=config[instance], key="cpu_burst_add"
+                    data=config[instance],
+                    key="cpu_burst_add",
+                    full_config=config,
+                    key_value=config[instance].get("cpu_burst_add"),
+                    full_config_flattened=config_flattened,
                 )
                 # we could probably have a separate error message if there's a comment that doesn't match
                 # the ack pattern, but that seems like overkill - especially for something that could cause
@@ -978,6 +1064,191 @@ def validate_cpu_burst(service_path: str) -> bool:
     return returncode
 
 
+def _check_smartstack_name_length_envoy(service: str, namespace: str) -> None:
+    """Ensures that Smartstack service name and namespace does not
+    exceed the limit on the length of Envoy's listener names
+    """
+    if len(service) + len(namespace) > MAX_ENVOY_NAME_LEN:
+        raise ValueError(
+            "Service name and namespace exceeds max listener name length in Envoy. Note that the full listener name "
+            'is "{}.{}.listener". Please rename so that the combined length of the service name and namespace does '
+            "not exceed {} characters".format(service, namespace, MAX_ENVOY_NAME_LEN),
+        )
+
+
+def _check_smartstack_name_length(service: str, namespace: str) -> None:
+    """Ensure that Smartstack name does not
+    exceed limits on HAProxy socket name
+    """
+    if len(service + namespace) > MAX_SMARTSTACK_NAME_LEN:
+        socket_name = "/var/run/synapse/sockets/{}.{}.sock.LONGPID.tmp".format(
+            service,
+            namespace,
+        )
+        raise ValueError(
+            "Name exceeds max socket name length. Note that the full socket name under the HAProxy naming scheme "
+            'is "{}". Please rename so that the combined length of the service name and namespace does not '
+            "exceed {} characters".format(socket_name, MAX_SMARTSTACK_NAME_LEN),
+        )
+
+
+@lru_cache()
+def _get_etc_services() -> list[str]:
+    with open("/etc/services") as f:
+        return f.read().splitlines()
+
+
+@lru_cache()
+def _get_etc_services_entry(port_lookup: int) -> str | None:
+    entries = _get_etc_services()
+    for entry in entries:
+        try:
+            service = entry.split()[0]
+            port = entry.split()[1]
+            if port.startswith("%s/" % str(port_lookup)):
+                return service
+        except IndexError:
+            continue
+    return None
+
+
+def _check_proxy_port_in_use(service: str, namespace: str, port: int) -> bool:
+    if port is None:
+        return False
+
+    # TODO(luisp): this should probably check the distributed /nail/etc/services
+    # smartstack.yamls OR we should more automatically manage /etc/services
+    etc_services_entry = _get_etc_services_entry(port)
+    if etc_services_entry is None:
+        return False
+    elif f"{service}.{namespace}" == etc_services_entry:
+        return False
+    else:
+        raise ValueError(
+            (
+                "port {} is already in use by {} according to /etc/services, it cannot be used by "
+                "{}.{}. Please either pick a different port or update /etc/services via puppet"
+            ).format(port, etc_services_entry, service, namespace),
+        )
+
+
+def _check_advertise_discover(
+    smartstack_data: dict[str, Any]
+) -> None:  # XXX: we should use a TypedDict here
+    """Need to ensure a few properties about smartstack files
+    1) discover is a member of advertise
+    2) discovery and advertise contain valid locations
+    3) extra_advertise contains valid locations
+    4) rhs of extra_advertise are >= discover type
+    """
+
+    def assert_valid_type(location_type: str) -> None:
+        if location_type not in available_location_types():
+            raise ValueError(
+                'Location type "{}" not a valid Yelp location type'.format(
+                    location_type,
+                ),
+            )
+
+    def assert_valid_location(location_string: str) -> None:
+        try:
+            typ, loc = location_string.split(":")
+            assert len(convert_location_type(loc, typ, typ)) == 1
+        except Exception:
+            raise ValueError(
+                'Location string "{}" not a valid Yelp location'.format(
+                    location_string,
+                ),
+            )
+
+    advertise = smartstack_data.get("advertise", ["region"])
+    discover = smartstack_data.get("discover", "region")
+    if discover not in advertise:
+        raise ValueError(
+            'discover key "{}" not a member of advertise "{}"'.format(
+                discover,
+                advertise,
+            ),
+        )
+    for location_type in [discover] + advertise:
+        assert_valid_type(location_type)
+
+    extra_advertisements = smartstack_data.get("extra_advertise", {})
+    for source, destinations in extra_advertisements.items():
+        assert_valid_location(source)
+        for destination in destinations:
+            assert_valid_location(destination)
+            dest_type = destination.split(":")[0]
+            if compare_types(dest_type, discover) > 0:
+                raise ValueError(
+                    'Right hand side "{}" less general than discover type "{}". Your advertisement would potentially '
+                    "result in more hosts seeing your service than intended. Please change the type of your RHS to be "
+                    ">= the discover type".format(destination, discover),
+                )
+
+
+def _check_smartstack_valid_proxy(proxied_through: str, soa_dir: str) -> None:
+    """Checks whether its parameter is a valid Smartstack namespace. Can be used for proxied_through or clb_proxy."""
+    proxy_service, proxy_namespace = proxied_through.split(".")
+    proxy_smartstack_filename = Path(soa_dir) / proxy_service / "smartstack.yaml"
+    try:
+        yaml_data = get_config_file_dict(proxy_smartstack_filename)
+        if proxy_namespace not in yaml_data:
+            raise ValueError(
+                f"{proxied_through} is not a valid Smartstack namespace to proxy through: "
+                f"{proxy_namespace} not found in {proxy_smartstack_filename}.",
+            )
+    except FileNotFoundError:
+        raise ValueError(
+            f"{proxied_through} is not a valid Smartstack namespace to proxy through: "
+            f"{proxy_smartstack_filename} does not exist.",
+        )
+
+
+def _check_smartstack_proxied_through(
+    smartstack_data: dict[str, Any],
+    soa_dir: str,
+) -> None:  # XXX: we should use a TypedDict here
+    """Checks the proxied_through field of a Smartstack namespace refers to another valid Smartstack namespace"""
+    if "proxied_through" not in smartstack_data:
+        return
+
+    proxied_through = smartstack_data["proxied_through"]
+    _check_smartstack_valid_proxy(proxied_through, soa_dir)
+
+
+def validate_smartstack(service_path: str) -> bool:
+    if not os.path.exists(os.path.join(service_path, "smartstack.yaml")):
+        # not every service is mesh-registered, exit early if this is the case
+        return True
+
+    config = get_config_file_dict(os.path.join(service_path, "smartstack.yaml"))
+    if not config:
+        print(
+            failure(
+                "smartstack.yaml is empty - if this service is not mesh-registered, please remove this file.",
+                "http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html",
+            )
+        )
+        return False
+
+    soa_dir, service = path_to_soa_dir_service(service_path)
+    for namespace, namespace_config in config.items():
+        # XXX(luisp): these should all really either return bools or be enforced in the schema
+        # ...i'm mostly leaving these as-is since i'm trying to remove some internal code that
+        # duplicates a bunch of paasta validate checks (i.e., py-gitolite)
+        _check_smartstack_name_length_envoy(service, namespace)
+        if namespace_config["proxy_port"]:
+            _check_smartstack_name_length(service, namespace)
+        proxy_port = namespace_config["proxy_port"]
+        _check_proxy_port_in_use(service, namespace, proxy_port)
+        _check_advertise_discover(namespace_config)
+        _check_smartstack_proxied_through(namespace_config, soa_dir)
+
+    print(success("All SmartStack configs are valid"))
+    return True
+
+
 def paasta_validate_soa_configs(
     service: str, service_path: str, verbose: bool = False
 ) -> bool:
@@ -1000,6 +1271,7 @@ def paasta_validate_soa_configs(
         validate_secrets,
         validate_min_max_instances,
         validate_cpu_burst,
+        validate_smartstack,
     ]
 
     # NOTE: we're explicitly passing a list comprehension to all()
@@ -1013,8 +1285,13 @@ def paasta_validate(args):
 
     :param args: argparse.Namespace obj created from sys.args by cli
     """
-    service_path = get_service_path(args.service, args.yelpsoa_config_root)
     service = args.service or guess_service_name()
+    service_path = get_service_path(service, args.yelpsoa_config_root)
+
+    # not much we can do if we have no path to inspect ;)
+    if not service_path:
+        return 1
+
     if not paasta_validate_soa_configs(service, service_path, args.verbose):
         print("Invalid configs found. Please try again.")
         return 1

paasta_tools/cli/cmds/wait_for_deployment.py

@@ -102,6 +102,8 @@ def add_subparser(subparsers):
         help="Name of the service which you wish to wait for deployment. "
         'Leading "services-" will be stripped.',
         required=True,
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     ).completer = lazy_choices_completer(list_services)
     list_parser.add_argument(
         "-t",

paasta_tools/cli/schemas/autoscaling_schema.json

@@ -10,8 +10,9 @@
                     "cpu",
                     "piscina",
                     "gunicorn",
-                    "arbitrary_promql",
-                    "active-requests"
+                    "active-requests",
+                    "arbitrary-promql",
+                    "worker-load"
                 ]
             },
             "decision_policy": {

paasta_tools/cli/schemas/eks_schema.json

@@ -1,6 +1,6 @@
 {
     "$schema": "http://json-schema.org/draft-04/schema#",
-    "description": "http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html#kubernetes-clustername-yaml",
+    "description": "http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html#eks-clustername-yaml",
     "type": "object",
     "additionalProperties": false,
     "minProperties": 1,
@@ -955,6 +955,28 @@
                             },
                             "max_skew": {
                                 "type": "integer"
+                            },
+                            "match_label_keys": {
+                                "type": "array",
+                                "items": {
+                                    "allOf": [
+                                        {
+                                            "type": "string",
+                                            "pattern": "^([a-zA-Z0-9]([-a-zA-Z0-9_.]*[a-zA-Z0-9])?/)?[a-zA-Z0-9]([-a-zA-Z0-9_.]*[a-zA-Z0-9])?$",
+                                            "maxLength": 63
+                                        },
+                                        {
+                                            "not": {
+                                                "enum": [
+                                                    "paasta.yelp.com/service",
+                                                    "paasta.yelp.com/instance"
+                                                ]
+                                            }
+                                        }
+                                    ],
+                                    "$comment": "Valid Kubernetes label key: optional prefix (DNS subdomain) followed by '/' and name segment (max 63 chars each)"
+                                },
+                                "uniqueItems": true
                             }
                         },
                         "required": []

paasta_tools/cli/schemas/smartstack_schema.json

@@ -62,6 +62,11 @@
                 "minimum": 0,
                 "maximum": 60000
             },
+            "timeout_client_ms": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 86400000
+            },
             "timeout_server_ms": {
                 "type": "integer",
                 "minimum": 0,
@@ -132,6 +137,10 @@
                     "CLUSTER_PROVIDED"
                 ]
             },
+            "choice_count": {
+                "type": "integer",
+                "minimum": 1
+            },
             "chaos": {
                 "type": "object",
                 "additionalProperties": {
@@ -156,6 +165,9 @@
             "proxied_through": {
                 "type": "string"
             },
+            "clb_proxy": {
+                "type": "string"
+            },
             "fixed_delay": {
                 "type": "object",
                 "additionalProperties": {

paasta_tools/cli/utils.py

@@ -577,7 +577,8 @@ def lazy_choices_completer(list_func):
 
 def figure_out_service_name(args, soa_dir=DEFAULT_SOA_DIR):
     """Figures out and validates the input service name"""
-    service = args.service or guess_service_name()
+    # most cmds should be doing this rstrip already - but just in case this is called from some other path...
+    service = args.service.rstrip("/") if args.service else guess_service_name()
     try:
         validate_service_name(service, soa_dir=soa_dir)
     except NoSuchService as service_not_found:

paasta_tools/contrib/paasta_update_soa_memcpu.py

@@ -181,6 +181,9 @@ def get_report_from_splunk(creds, app, filename, criteria_filter):
             "date": d["result"]["_time"].split(" ")[0],
             "instance": criteria.split(" ")[2],
             "money": d["result"].get("estimated_monthly_savings", 0),
+            "old_cpus": d["result"].get("current_cpus"),
+            "old_disk": d["result"].get("current_disk"),
+            "old_mem": d["result"].get("current_mem"),
             "owner": d["result"].get("service_owner", "Unavailable"),
             "project": d["result"].get("project", "Unavailable"),
             "service": criteria.split(" ")[0],
@@ -192,24 +195,17 @@ def get_report_from_splunk(creds, app, filename, criteria_filter):
             "max_instances": d["result"].get("suggested_max_instances"),
             "mem": d["result"].get("suggested_mem"),
             "min_instances": d["result"].get("suggested_min_instances"),
-            "old_cpus": d["result"].get("current_cpus"),
-            "old_disk": d["result"].get("current_disk"),
-            "old_mem": d["result"].get("current_mem"),
         }
 
         # the report we get is all strings, so we need to convert them to the right types
         field_conversions = {
-            "current_cpus": float,
-            "suggested_cpu_burst_add": float,
-            "suggested_cpus": float,
-            "suggested_disk": int,
-            "suggested_hacheck_cpus": float,
-            "suggested_max_instances": int,
-            "suggested_mem": int,
-            "suggested_min_instances": int,
-            # not quite sure why these are floats...they're ints in soaconfigs
-            "current_disk": _force_str_to_int,
-            "current_mem": _force_str_to_int,
+            "cpus": float,
+            "cpu_burst_add": float,
+            "disk": int,
+            "hacheck_cpus": float,
+            "max_instances": int,
+            "mem": int,
+            "min_instances": int,
         }
 
         # merge results if we've already seen rows for this service

paasta_tools/generate_deployments_for_service.py

@@ -101,6 +101,8 @@ def parse_args() -> argparse.Namespace:
         "--service",
         required=True,
         help="Service name to make the deployments.json for",
+        # strip any potential trailing / for folks tab-completing directories
+        type=lambda x: x.rstrip("/"),
     )
     args = parser.parse_args()
     return args

paasta_tools/instance/hpa_metrics_parser.py

@@ -1,8 +1,6 @@
 from typing import Optional
 
-from kubernetes.client.models.v2beta2_object_metric_status import (
-    V2beta2ObjectMetricStatus,
-)
+from kubernetes.client import V2ObjectMetricStatus
 from mypy_extensions import TypedDict
 
 
@@ -102,7 +100,7 @@ class HPAMetricsParser:
         )
 
     def parse_object_metric(
-        self, metric_spec: V2beta2ObjectMetricStatus, status: HPAMetricsDict
+        self, metric_spec: V2ObjectMetricStatus, status: HPAMetricsDict
     ) -> None:
         status["name"] = metric_spec.metric.name
         status["target_value"] = (
@@ -112,7 +110,7 @@ class HPAMetricsParser:
         )
 
     def parse_object_metric_current(
-        self, metric_spec: V2beta2ObjectMetricStatus, status: HPAMetricsDict
+        self, metric_spec: V2ObjectMetricStatus, status: HPAMetricsDict
     ) -> None:
         status["name"] = metric_spec.metric.name
         status["current_value"] = (