paasta-tools 1.34.0__tar.gz → 1.35.1__tar.gz

{paasta_tools-1.34.0/paasta_tools.egg-info → paasta_tools-1.35.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: paasta-tools
-Version: 1.34.0
+Version: 1.35.1
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com

{paasta_tools-1.34.0 → paasta_tools-1.35.1}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.34.0"
+__version__ = "1.35.1"

{paasta_tools-1.34.0 → paasta_tools-1.35.1}/paasta_tools/cli/cmds/remote_run.py

@@ -295,11 +295,19 @@ def paasta_remote_run_stop(
 
 
 def add_common_args_to_parser(parser: argparse.ArgumentParser):
+    def _validate_service_name(name: str) -> str:
+        if name not in _list_services_and_toolboxes():
+            raise ValueError(f"{name} is not a known service name")
+        return name
+
     service_arg = parser.add_argument(
         "-s",
         "--service",
         help="The name of the service you wish to inspect. Required.",
         required=True,
+        # not using `choices` for validation to avoid the help text
+        # for the command being incredibly large
+        type=_validate_service_name,
     )
     service_arg.completer = lazy_choices_completer(_list_services_and_toolboxes)  # type: ignore
     instance_or_toolbox = parser.add_mutually_exclusive_group()
@@ -323,6 +331,7 @@ def add_common_args_to_parser(parser: argparse.ArgumentParser):
         "--cluster",
         help="The name of the cluster you wish to run your task on. Required.",
         required=True,
+        choices=list_clusters(),
     )
     cluster_arg.completer = lazy_choices_completer(list_clusters)  # type: ignore
 

{paasta_tools-1.34.0 → paasta_tools-1.35.1}/paasta_tools/cli/cmds/validate.py

@@ -22,6 +22,7 @@ from datetime import datetime
 from functools import lru_cache
 from functools import partial
 from glob import glob
+from pathlib import Path
 from typing import Any
 from typing import Callable
 from typing import cast
@@ -34,6 +35,9 @@ from typing import Union
 
 import pytz
 from croniter import croniter
+from environment_tools.type_utils import available_location_types
+from environment_tools.type_utils import compare_types
+from environment_tools.type_utils import convert_location_type
 from jsonschema import Draft4Validator
 from jsonschema import exceptions
 from jsonschema import FormatChecker
@@ -121,6 +125,7 @@ SCHEMA_TYPES = {
     "rollback",  # automatic rollbacks during deployments
     "tron",  # batch workloads
     "eks",  # eks workloads
+    "smartstack",  # mesh configs
     "autotuned_defaults/kubernetes",
     "autotuned_defaults/cassandracluster",
 }
@@ -171,6 +176,26 @@ INVALID_AUTOSCALING_FIELDS = {
     METRICS_PROVIDER_PROMQL: {"desired_active_requests_per_replica"},
 }
 
+# Listener names in Envoy cannot exceed 128 characters and use the
+# following format:
+#    service_name.namespace.listener
+# This naming scheme leaves 128 - 10 = 118 characters
+# for the Smartstack service name and namespace.
+# See COREBACK-6303 for more context.
+MAX_ENVOY_NAME_LEN = 118
+
+# Socket names cannot exceed 108 characters, and the longest socket
+# name generated by HAProxy uses the following format:
+#    /var/run/synapse/sockets/service_name.namespace.LONGPID.sock.tmp
+# This naming scheme leaves 108 - 43 = 65 characters combined for the
+# Smartstack service name and namespace. We leave a generous buffer
+# to arrive at a maximum name length of 55, in case e.g. the .sock
+# suffix is renamed to a longer name for certain sockets.
+# See SMTSTK-204 for more context.
+# NOTE: the above is mostly still true - but the path we use is now /var/run/envoy/sockets/...
+# so we may want to adjust this a tad in the future ;)
+MAX_SMARTSTACK_NAME_LEN = 55
+
 
 class ConditionConfig(TypedDict, total=False):
     """
@@ -983,6 +1008,191 @@ def validate_cpu_burst(service_path: str) -> bool:
     return returncode
 
 
+def _check_smartstack_name_length_envoy(service: str, namespace: str) -> None:
+    """Ensures that Smartstack service name and namespace does not
+    exceed the limit on the length of Envoy's listener names
+    """
+    if len(service) + len(namespace) > MAX_ENVOY_NAME_LEN:
+        raise ValueError(
+            "Service name and namespace exceeds max listener name length in Envoy. Note that the full listener name "
+            'is "{}.{}.listener". Please rename so that the combined length of the service name and namespace does '
+            "not exceed {} characters".format(service, namespace, MAX_ENVOY_NAME_LEN),
+        )
+
+
+def _check_smartstack_name_length(service: str, namespace: str) -> None:
+    """Ensure that Smartstack name does not
+    exceed limits on HAProxy socket name
+    """
+    if len(service + namespace) > MAX_SMARTSTACK_NAME_LEN:
+        socket_name = "/var/run/synapse/sockets/{}.{}.sock.LONGPID.tmp".format(
+            service,
+            namespace,
+        )
+        raise ValueError(
+            "Name exceeds max socket name length. Note that the full socket name under the HAProxy naming scheme "
+            'is "{}". Please rename so that the combined length of the service name and namespace does not '
+            "exceed {} characters".format(socket_name, MAX_SMARTSTACK_NAME_LEN),
+        )
+
+
+@lru_cache()
+def _get_etc_services() -> list[str]:
+    with open("/etc/services") as f:
+        return f.read().splitlines()
+
+
+@lru_cache()
+def _get_etc_services_entry(port_lookup: int) -> str | None:
+    entries = _get_etc_services()
+    for entry in entries:
+        try:
+            service = entry.split()[0]
+            port = entry.split()[1]
+            if port.startswith("%s/" % str(port_lookup)):
+                return service
+        except IndexError:
+            continue
+    return None
+
+
+def _check_proxy_port_in_use(service: str, namespace: str, port: int) -> bool:
+    if port is None:
+        return False
+
+    # TODO(luisp): this should probably check the distributed /nail/etc/services
+    # smartstack.yamls OR we should more automatically manage /etc/services
+    etc_services_entry = _get_etc_services_entry(port)
+    if etc_services_entry is None:
+        return False
+    elif f"{service}.{namespace}" == etc_services_entry:
+        return False
+    else:
+        raise ValueError(
+            (
+                "port {} is already in use by {} according to /etc/services, it cannot be used by "
+                "{}.{}. Please either pick a different port or update /etc/services via puppet"
+            ).format(port, etc_services_entry, service, namespace),
+        )
+
+
+def _check_advertise_discover(
+    smartstack_data: dict[str, Any]
+) -> None:  # XXX: we should use a TypedDict here
+    """Need to ensure a few properties about smartstack files
+    1) discover is a member of advertise
+    2) discovery and advertise contain valid locations
+    3) extra_advertise contains valid locations
+    4) rhs of extra_advertise are >= discover type
+    """
+
+    def assert_valid_type(location_type: str) -> None:
+        if location_type not in available_location_types():
+            raise ValueError(
+                'Location type "{}" not a valid Yelp location type'.format(
+                    location_type,
+                ),
+            )
+
+    def assert_valid_location(location_string: str) -> None:
+        try:
+            typ, loc = location_string.split(":")
+            assert len(convert_location_type(loc, typ, typ)) == 1
+        except Exception:
+            raise ValueError(
+                'Location string "{}" not a valid Yelp location'.format(
+                    location_string,
+                ),
+            )
+
+    advertise = smartstack_data.get("advertise", ["region"])
+    discover = smartstack_data.get("discover", "region")
+    if discover not in advertise:
+        raise ValueError(
+            'discover key "{}" not a member of advertise "{}"'.format(
+                discover,
+                advertise,
+            ),
+        )
+    for location_type in [discover] + advertise:
+        assert_valid_type(location_type)
+
+    extra_advertisements = smartstack_data.get("extra_advertise", {})
+    for source, destinations in extra_advertisements.items():
+        assert_valid_location(source)
+        for destination in destinations:
+            assert_valid_location(destination)
+            dest_type = destination.split(":")[0]
+            if compare_types(dest_type, discover) > 0:
+                raise ValueError(
+                    'Right hand side "{}" less general than discover type "{}". Your advertisement would potentially '
+                    "result in more hosts seeing your service than intended. Please change the type of your RHS to be "
+                    ">= the discover type".format(destination, discover),
+                )
+
+
+def _check_smartstack_valid_proxy(proxied_through: str, soa_dir: str) -> None:
+    """Checks whether its parameter is a valid Smartstack namespace. Can be used for proxied_through or clb_proxy."""
+    proxy_service, proxy_namespace = proxied_through.split(".")
+    proxy_smartstack_filename = Path(soa_dir) / proxy_service / "smartstack.yaml"
+    try:
+        yaml_data = get_config_file_dict(proxy_smartstack_filename)
+        if proxy_namespace not in yaml_data:
+            raise ValueError(
+                f"{proxied_through} is not a valid Smartstack namespace to proxy through: "
+                f"{proxy_namespace} not found in {proxy_smartstack_filename}.",
+            )
+    except FileNotFoundError:
+        raise ValueError(
+            f"{proxied_through} is not a valid Smartstack namespace to proxy through: "
+            f"{proxy_smartstack_filename} does not exist.",
+        )
+
+
+def _check_smartstack_proxied_through(
+    smartstack_data: dict[str, Any],
+    soa_dir: str,
+) -> None:  # XXX: we should use a TypedDict here
+    """Checks the proxied_through field of a Smartstack namespace refers to another valid Smartstack namespace"""
+    if "proxied_through" not in smartstack_data:
+        return
+
+    proxied_through = smartstack_data["proxied_through"]
+    _check_smartstack_valid_proxy(proxied_through, soa_dir)
+
+
+def validate_smartstack(service_path: str) -> bool:
+    if not os.path.exists(os.path.join(service_path, "smartstack.yaml")):
+        # not every service is mesh-registered, exit early if this is the case
+        return True
+
+    config = get_config_file_dict(os.path.join(service_path, "smartstack.yaml"))
+    if not config:
+        print(
+            failure(
+                "smartstack.yaml is empty - if this service is not mesh-registered, please remove this file.",
+                "http://paasta.readthedocs.io/en/latest/yelpsoa_configs.html",
+            )
+        )
+        return False
+
+    soa_dir, service = path_to_soa_dir_service(service_path)
+    for namespace, namespace_config in config.items():
+        # XXX(luisp): these should all really either return bools or be enforced in the schema
+        # ...i'm mostly leaving these as-is since i'm trying to remove some internal code that
+        # duplicates a bunch of paasta validate checks (i.e., py-gitolite)
+        _check_smartstack_name_length_envoy(service, namespace)
+        if namespace_config["proxy_port"]:
+            _check_smartstack_name_length(service, namespace)
+        proxy_port = namespace_config["proxy_port"]
+        _check_proxy_port_in_use(service, namespace, proxy_port)
+        _check_advertise_discover(namespace_config)
+        _check_smartstack_proxied_through(namespace_config, soa_dir)
+
+    print(success("All SmartStack configs are valid"))
+    return True
+
+
 def paasta_validate_soa_configs(
     service: str, service_path: str, verbose: bool = False
 ) -> bool:
@@ -1005,6 +1215,7 @@ def paasta_validate_soa_configs(
         validate_secrets,
         validate_min_max_instances,
         validate_cpu_burst,
+        validate_smartstack,
     ]
 
     # NOTE: we're explicitly passing a list comprehension to all()
@@ -1018,8 +1229,13 @@ def paasta_validate(args):
 
     :param args: argparse.Namespace obj created from sys.args by cli
     """
-    service_path = get_service_path(args.service, args.yelpsoa_config_root)
     service = args.service or guess_service_name()
+    service_path = get_service_path(service, args.yelpsoa_config_root)
+
+    # not much we can do if we have no path to inspect ;)
+    if not service_path:
+        return 1
+
     if not paasta_validate_soa_configs(service, service_path, args.verbose):
         print("Invalid configs found. Please try again.")
         return 1

{paasta_tools-1.34.0 → paasta_tools-1.35.1}/paasta_tools/cli/schemas/smartstack_schema.json

@@ -62,6 +62,11 @@
                 "minimum": 0,
                 "maximum": 60000
             },
+            "timeout_client_ms": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 86400000
+            },
             "timeout_server_ms": {
                 "type": "integer",
                 "minimum": 0,
@@ -132,6 +137,10 @@
                     "CLUSTER_PROVIDED"
                 ]
             },
+            "choice_count": {
+                "type": "integer",
+                "minimum": 1
+            },
             "chaos": {
                 "type": "object",
                 "additionalProperties": {
@@ -156,6 +165,9 @@
             "proxied_through": {
                 "type": "string"
             },
+            "clb_proxy": {
+                "type": "string"
+            },
             "fixed_delay": {
                 "type": "object",
                 "additionalProperties": {

{paasta_tools-1.34.0 → paasta_tools-1.35.1/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: paasta-tools
-Version: 1.34.0
+Version: 1.35.1
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com