paasta-tools 1.26.0__tar.gz → 1.27.0__tar.gz

{paasta_tools-1.26.0/paasta_tools.egg-info → paasta_tools-1.27.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: paasta-tools
-Version: 1.26.0
+Version: 1.27.0
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com

{paasta_tools-1.26.0 → paasta_tools-1.27.0}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.26.0"
+__version__ = "1.27.0"

{paasta_tools-1.26.0 → paasta_tools-1.27.0}/paasta_tools/api/api_docs/swagger.json

@@ -322,8 +322,14 @@
                                 },
                                 "min_instances": {
                                     "type": "integer",
+                                    "x-nullable": true,
                                     "description": "Minimum number of instances to run"
                                 },
+                                "max_instances": {
+                                    "type": "integer",
+                                    "x-nullable": true,
+                                    "description": "Maximum number of instances to run"
+                                },
                                 "expire_after": {
                                     "type": "number",
                                     "format": "float",
@@ -1724,17 +1730,24 @@
             "properties": {
                 "min_instances": {
                     "type": "integer",
-                    "description": "Minimum number of instances to run",
-                    "minimum": 1
+                    "x-nullable": true,
+                    "description": "Minimum number of instances to run"
+                },
+                "max_instances": {
+                    "type": "integer",
+                    "x-nullable": true,
+                    "description": "Maximum number of instances to run"
                 },
                 "expire_after": {
                     "type": "number",
                     "format": "float",
-                    "description": "Unix timestamp when this override is no longer valid"
+                    "description": "Unix timestamp when this override is no longer valid",
+                    "minimum": 1
                 }
             },
             "required": [
                 "min_instances",
+                "max_instances",
                 "expire_after"
             ]
         },

{paasta_tools-1.26.0 → paasta_tools-1.27.0}/paasta_tools/api/views/autoscaler.py

@@ -136,7 +136,7 @@ def set_autoscaling_override(request):
     Required parameters:
     - service: The service name
     - instance: The instance name
-    - min_instances: The minimum number of instances to enforce
+    - min_instances AND/OR max_instances: The minimum and/or maximum number of instances to enforce
     - expires_after: unix timestamp after which the override is no longer valid
     """
     service = request.swagger_data.get("service")
@@ -156,24 +156,29 @@ def set_autoscaling_override(request):
 
     json_body = request.swagger_data.get("json_body", {})
     min_instances_override = json_body.get("min_instances")
-    expire_after = json_body.get("expire_after")
+    max_instances_override = json_body.get("max_instances")
+    expire_after = json_body["expire_after"]
 
-    if not isinstance(min_instances_override, int) or min_instances_override < 1:
-        raise ApiFailure("min_instances must be a positive integer", 400)
+    # ideally we'd enforce this in our api spec - but since we're using both swagger and oapi, that's not quite so simple
+    if not min_instances_override and not max_instances_override:
+        raise ApiFailure(
+            "At least one of min_instances or max_instances must be provided", 400
+        )
 
-    if not expire_after:
-        raise ApiFailure("expire_after is required", 400)
+    # we can't currently enforce this in the API spec since the generator seems to have issues generating a validator
+    # for a nullable integer field with a minimum value as we get errors like:
+    # `TypeError: '<' not supported between instances of 'NoneType' and 'int'`
+    # that said, unless someone is manually crafting requests, this should never happen
+    # since the CLI includes a check for this as well
+    if min_instances_override is not None and min_instances_override < 1:
+        raise ApiFailure("min_instances must be a positive integer", 400)
+    if max_instances_override is not None and max_instances_override < 1:
+        raise ApiFailure("max_instances must be a positive integer", 400)
 
-    max_instances = instance_config.get_max_instances()
-    if max_instances is None:
+    # otherwise, we could have folks accidentally enable autoscaling (using cpu) for a non-autoscaled
+    if instance_config.is_autoscaling_enabled() is None:
         raise ApiFailure(f"Autoscaling is not enabled for {service}.{instance}", 400)
 
-    if max_instances < min_instances_override:
-        raise ApiFailure(
-            f"min_instances ({min_instances_override}) cannot be greater than max_instances ({max_instances})",
-            400,
-        )
-
     configmap, created = get_or_create_autoscaling_overrides_configmap()
     if created:
         log.info("Created new autoscaling overrides ConfigMap")
@@ -182,23 +187,54 @@ def set_autoscaling_override(request):
     if not configmap.data:
         configmap.data = {}
 
-    override_data = {
-        "min_instances": min_instances_override,
-        "created_at": datetime.now(timezone.utc).isoformat(),
-        # NOTE: we may want to also allow setting a max_instances override in the future, but if we do that
-        # we'd probably want to force folks to either set one or both and share the same expiration time
-        "expire_after": expire_after,
-    }
-
     service_instance = f"{service}.{instance}"
     existing_overrides = (
         json.loads(configmap.data[service_instance])
         if service_instance in configmap.data
         else {}
     )
-    merged_overrides = {**existing_overrides, **override_data}
+
+    # this is slightly funky as there's a hierarchy of what value to pick:
+    # 1. the override value provided in the request - as this should override any existing value
+    # 2. the existing override value in the configmap - this is used if we previously only set an override for min or
+    #    max instances and now want to set a value for the other
+    # 3. the value from the instance config - this is used as a final fallback since this means no override is present
+    min_instances = (
+        min_instances_override
+        or existing_overrides.get("min_instances")
+        or instance_config.get_min_instances()
+    )
+    max_instances = (
+        max_instances_override
+        or existing_overrides.get("max_instances")
+        or instance_config.get_max_instances()
+    )
+
+    # NOTE: the max_instances check is unnecessary here, but type-checkers can't see that the is_autoscaling_enabled()
+    # check above ensures that max_instances is not None
+    if max_instances is None or max_instances < min_instances:
+        raise ApiFailure(
+            f"min_instances ({min_instances_override}) cannot be greater than max_instances ({max_instances})",
+            400,
+        )
+
+    override_data = {
+        "min_instances": min_instances_override,
+        "max_instances": max_instances_override,
+        "created_at": datetime.now(timezone.utc).isoformat(),
+        "expire_after": expire_after,
+    }
+
+    # NOTE: we need to strip out null values from the incoming overrides since otherwise we'd remove existing overrides
+    # (since if only --set-min or --set-max is provided, the other will be sent as None)
+    merged_overrides = {
+        **existing_overrides,
+        **{k: v for k, v in override_data.items() if v is not None},
+    }
     serialized_overrides = json.dumps(merged_overrides)
 
+    # note: we can only append with patching - if we ever want to remove overrides outside of the cleanup cronjob,
+    # we would need to change this to use replace_namespaced_configmap
     patch_namespaced_configmap(
         name=AUTOSCALING_OVERRIDES_CONFIGMAP_NAME,
         namespace=AUTOSCALING_OVERRIDES_CONFIGMAP_NAMESPACE,
@@ -213,6 +249,7 @@ def set_autoscaling_override(request):
         "instance": instance,
         "cluster": cluster,
         "min_instances": min_instances_override,
+        "max_instances": max_instances_override,
         "expire_after": expire_after,
         "status": "SUCCESS",
     }

{paasta_tools-1.26.0 → paasta_tools-1.27.0}/paasta_tools/cli/cmds/autoscale.py

@@ -75,10 +75,18 @@ def add_subparser(subparsers):
         else autoscale_parser.error("Minimum instances must be >= 1"),
         default=None,
     )
+    override_group.add_argument(
+        "--set-max",
+        help="Set the maximum number of replicas (must be >= 1). Requires --for parameter.",
+        type=lambda x: int(x)
+        if int(x) >= 1
+        else autoscale_parser.error("Maximum instances must be >= 1"),
+        default=None,
+    )
     override_group.add_argument(
         "--for",
         dest="duration",
-        help="Duration for the temporary override (e.g. '3h', '30m'). Required when using --set-min.",
+        help="Duration for the temporary override (e.g. '3h', '30m'). Required when using --set-min and/or --set-max.",
         default=None,
     )
 
@@ -113,22 +121,24 @@ def paasta_autoscale(args):
     log.setLevel(logging.DEBUG)
     service = figure_out_service_name(args)
 
-    if args.set_min is not None and not args.duration:
+    if (args.set_min is not None or args.set_max is not None) and not args.duration:
         print(
             PaastaColors.yellow(
-                "WARNING: --set-min requires --for parameter to specify duration - defaulting to 30m"
+                "WARNING: --set-min/--set-max usage requires --for parameter to specify duration - defaulting to 30m"
             )
         )
         args.duration = "30m"
 
-    if args.duration is not None and args.set_min is None:
-        print(PaastaColors.red("Error: --for requires --set-min parameter"))
+    if args.duration is not None and args.set_min is None and args.set_max is None:
+        print(
+            PaastaColors.red("Error: --for requires --set-min or --set-max parameter")
+        )
         return 1
 
-    if args.set is not None and args.set_min is not None:
+    if args.set is not None and args.set_min is not None and args.set_max is not None:
         print(
             PaastaColors.red(
-                "Error: Cannot use both --set and --set-min at the same time"
+                "Error: Cannot use both --set and --set-min or --set-max at the same time"
             )
         )
         return 1
@@ -159,9 +169,12 @@ def paasta_autoscale(args):
         print("Could not connect to paasta api. Maybe you misspelled the cluster?")
         return 1
 
+    # TODO: we should probably also make sure we set a couple other defaults - currently, it's possible for
+    # status/res/etc to be unbound in some code paths
+    err_reason = None
     try:
         # get current autoscaler count
-        if args.set is None and args.set_min is None:
+        if args.set is None and args.set_min is None and args.set_max is None:
             log.debug("Getting the current autoscaler count...")
             res, status, _ = api.autoscaler.get_autoscaler_count(
                 service=service, instance=args.instance, _return_http_data_only=False
@@ -187,7 +200,7 @@ def paasta_autoscale(args):
             )
 
         # set lower bound
-        elif args.set_min is not None:
+        elif args.set_min is not None or args.set_max is not None:
             duration_seconds = parse_duration_to_seconds(args.duration)
             if not duration_seconds:
                 print(
@@ -202,10 +215,11 @@ def paasta_autoscale(args):
             expiration_time = time.time() + duration_seconds
 
             log.debug(
-                f"Setting minimum instances to {args.set_min} for duration {args.duration}."
+                f"Sending the following overrides for duration {args.duration}: min_instances: {args.set_min}, max_instances: {args.set_max}."
             )
             msg = paastamodels.AutoscalingOverride(
                 min_instances=args.set_min,
+                max_instances=args.set_max,
                 expire_after=expiration_time,
             )
 
@@ -224,28 +238,35 @@ def paasta_autoscale(args):
             )
     except api.api_error as exc:
         status = exc.status
+        err_reason = exc.body
 
     if not 200 <= status <= 299:
         print(
             PaastaColors.red(
-                f"ERROR: '{args.instance}' is not configured to autoscale OR you set min_instances above the current max_instances, "
+                f"ERROR: '{args.instance}' is not configured to autoscale OR you set impossible {{min, max}}_instances, "
                 f"and `paasta autoscale` could not update it. "
                 f"If you want to be able to boost this service, please configure autoscaling for the service "
                 f"in its config file by setting min and max instances appropriately. Example: \n"
                 f"{args.instance}:\n"
                 f"     min_instances: 5\n"
-                f"     max_instances: 50"
+                f"     max_instances: 50\n"
+                f"{err_reason}"
             )
         )
         return 0
 
     log.debug(f"Res: {res} Http: {status}")
-    if not args.set_min:
+    if not args.set_min and not args.set_max:
         print(f"Desired instances: {res.desired_instances}")
-    elif args.set_min:
-        print(
-            f"Temporary override set for {args.service}.{args.instance} with minimum instances: {args.set_min}"
-        )
+    else:
+        if args.set_min:
+            print(
+                f"Temporary override set for {args.service}.{args.instance} with minimum instances: {args.set_min}"
+            )
+        if args.set_max:
+            print(
+                f"Temporary override set for {args.service}.{args.instance} with maximum instances: {args.set_max}"
+            )
         # folks using this might be in different timezones, so let's convert the expiration time to a few common ones
         # to make it extra clear when the override will expire
         epoch_time = datetime.fromtimestamp(res.expire_after)

{paasta_tools-1.26.0 → paasta_tools-1.27.0}/paasta_tools/kubernetes/application/controller_wrappers.py

@@ -301,7 +301,10 @@ class DeploymentWrapper(Application):
             kube_client=kube_client,
             namespace=self.item.metadata.namespace,
             min_instances_override=(
-                self.hpa_override["min_instances"] if self.hpa_override else None
+                self.hpa_override.get("min_instances") if self.hpa_override else None
+            ),
+            max_instances_override=(
+                self.hpa_override.get("max_instances") if self.hpa_override else None
             ),
         )
 

{paasta_tools-1.26.0 → paasta_tools-1.27.0}/paasta_tools/kubernetes_tools.py

@@ -325,6 +325,7 @@ class DatastoreCredentialsConfig(TypedDict, total=False):
 
 class HpaOverride(TypedDict):
     min_instances: int
+    max_instances: int
     expire_after: str
 
 
@@ -899,6 +900,7 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
         kube_client: KubeClient,
         namespace: str,
         min_instances_override: Optional[int] = None,
+        max_instances_override: Optional[int] = None,
     ) -> Optional[V2HorizontalPodAutoscaler]:
         # Returns None if an HPA should not be attached based on the config,
         # or the config is invalid.
@@ -914,7 +916,7 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
             return None
 
         min_replicas = min_instances_override or self.get_min_instances()
-        max_replicas = self.get_max_instances()
+        max_replicas = max_instances_override or self.get_max_instances()
         if min_replicas == 0 or max_replicas == 0:
             log.error(
                 f"Invalid value for min or max_instances on {name}: {min_replicas}, {max_replicas}"
@@ -2015,7 +2017,7 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
             )
             return None
 
-    def get_min_instances(self) -> Optional[int]:
+    def get_min_instances(self) -> int:
         return self.config_dict.get(
             "min_instances",
             1,
@@ -4690,7 +4692,9 @@ def get_namespaced_configmap(
 
 def patch_namespaced_configmap(
     name: str,
-    body: Dict[str, str],
+    # NOTE: passing a dict *seems* to work - but we should likely switch to passing a ConfigMap since that seems to be
+    # the most supported option
+    body: Union[V1ConfigMap, Dict[str, Any]],
     *,
     namespace: str,
     kube_client: KubeClient,

{paasta_tools-1.26.0 → paasta_tools-1.27.0}/paasta_tools/paastaapi/model/autoscaling_override.py

@@ -59,9 +59,6 @@ class AutoscalingOverride(ModelNormal):
     }
 
     validations = {
-        ('min_instances',): {
-            'inclusive_minimum': 1,
-        },
     }
 
     additional_properties_type = None
@@ -79,7 +76,8 @@ class AutoscalingOverride(ModelNormal):
                 and the value is attribute type.
         """
         return {
-            'min_instances': (int,),  # noqa: E501
+            'min_instances': (int, none_type,),  # noqa: E501
+            'max_instances': (int, none_type,),  # noqa: E501
             'expire_after': (float,),  # noqa: E501
         }
 
@@ -90,6 +88,7 @@ class AutoscalingOverride(ModelNormal):
 
     attribute_map = {
         'min_instances': 'min_instances',  # noqa: E501
+        'max_instances': 'max_instances',  # noqa: E501
         'expire_after': 'expire_after',  # noqa: E501
     }
 
@@ -105,11 +104,12 @@ class AutoscalingOverride(ModelNormal):
     ])
 
     @convert_js_args_to_python_args
-    def __init__(self, min_instances, expire_after, *args, **kwargs):  # noqa: E501
+    def __init__(self, min_instances, max_instances, expire_after, *args, **kwargs):  # noqa: E501
         """AutoscalingOverride - a model defined in OpenAPI
 
         Args:
-            min_instances (int): Minimum number of instances to run
+            min_instances (int, none_type): Minimum number of instances to run
+            max_instances (int, none_type): Maximum number of instances to run
             expire_after (float): Unix timestamp when this override is no longer valid
 
         Keyword Args:
@@ -169,6 +169,7 @@ class AutoscalingOverride(ModelNormal):
         self._visited_composed_classes = _visited_composed_classes + (self.__class__,)
 
         self.min_instances = min_instances
+        self.max_instances = max_instances
         self.expire_after = expire_after
         for var_name, var_value in kwargs.items():
             if var_name not in self.attribute_map and \

{paasta_tools-1.26.0 → paasta_tools-1.27.0}/paasta_tools/paastaapi/model/inline_response202.py

@@ -78,7 +78,8 @@ class InlineResponse202(ModelNormal):
         return {
             'service': (str,),  # noqa: E501
             'instance': (str,),  # noqa: E501
-            'min_instances': (int,),  # noqa: E501
+            'min_instances': (int, none_type,),  # noqa: E501
+            'max_instances': (int, none_type,),  # noqa: E501
             'expire_after': (float,),  # noqa: E501
             'status': (str,),  # noqa: E501
         }
@@ -92,6 +93,7 @@ class InlineResponse202(ModelNormal):
         'service': 'service',  # noqa: E501
         'instance': 'instance',  # noqa: E501
         'min_instances': 'min_instances',  # noqa: E501
+        'max_instances': 'max_instances',  # noqa: E501
         'expire_after': 'expire_after',  # noqa: E501
         'status': 'status',  # noqa: E501
     }
@@ -144,7 +146,8 @@ class InlineResponse202(ModelNormal):
                                 _visited_composed_classes = (Animal,)
             service (str): Service name. [optional]  # noqa: E501
             instance (str): Instance name. [optional]  # noqa: E501
-            min_instances (int): Minimum number of instances to run. [optional]  # noqa: E501
+            min_instances (int, none_type): Minimum number of instances to run. [optional]  # noqa: E501
+            max_instances (int, none_type): Maximum number of instances to run. [optional]  # noqa: E501
             expire_after (float): Unix timestamp after which the override is no longer valid. [optional]  # noqa: E501
             status (str): Status of the operation. [optional]  # noqa: E501
         """

{paasta_tools-1.26.0 → paasta_tools-1.27.0}/paasta_tools/setup_kubernetes_cr.py

@@ -49,6 +49,7 @@ from paasta_tools.utils import get_config_hash
 from paasta_tools.utils import get_git_sha_from_dockerurl
 from paasta_tools.utils import load_all_configs
 from paasta_tools.utils import load_system_paasta_config
+from paasta_tools.utils import NoDeploymentsAvailable
 
 
 log = logging.getLogger(__name__)
@@ -344,13 +345,19 @@ def reconcile_kubernetes_resource(
         if instance is not None and instance != inst:
             continue
         try:
-            soa_config = config_handler.loader(
-                service=service,
-                instance=inst,
-                cluster=cluster,
-                load_deployments=True,
-                soa_dir=DEFAULT_SOA_DIR,
-            )
+            try:
+                soa_config = config_handler.loader(
+                    service=service,
+                    instance=inst,
+                    cluster=cluster,
+                    load_deployments=True,
+                    soa_dir=DEFAULT_SOA_DIR,
+                )
+            except NoDeploymentsAvailable:
+                log.warning(
+                    f"No deployments available for {service}.{inst} in {cluster}, skipping"
+                )
+                continue
             git_sha = get_git_sha_from_dockerurl(soa_config.get_docker_url(), long=True)
             formatted_resource = format_custom_resource(
                 instance_config=config,

{paasta_tools-1.26.0 → paasta_tools-1.27.0}/paasta_tools/setup_kubernetes_job.py

@@ -255,27 +255,31 @@ def get_hpa_overrides(kube_client: KubeClient) -> Dict[str, Dict[str, HpaOverrid
 
     This function reads the "paasta-autoscaling-overrides" ConfigMap in the "paasta" namespace
     and extracts all valid (non-expired) overrides to return a dictionary mapping
-    service.instance pairs to override data (currently, just min_instances and when the
+    service.instance pairs to override data (currently, just {min,max_instances} and when the
     override should expire by).
 
     The incoming ConfigMap is expected to have the following format:
     {
         $SERVICE_A.$INSTANCE_A: {
             "min_instances": 2,
+            "m_instances": 2,
             "expire_after": "2023-10-01T00:00:00Z"
         },
         $SERVICE_A.$INSTANCE_B: {
             "min_instances": 3,
+            "max_instances": 10,
             "expire_after": "2023-10-01T00:00:00Z"
         },
             ...
         },
         $SERVICE_B.$INSTANCE_A: {
             "min_instances": 1,
+            "max_instances": 11,
             "expire_after": "2023-10-01T00:00:00Z"
         },
         $SERVICE_B.$INSTANCE_B: {
             "min_instances": 2,
+            "max_instances": 20,
             "expire_after": "2023-10-01T00:00:00Z"
         },
         ...
@@ -298,19 +302,21 @@ def get_hpa_overrides(kube_client: KubeClient) -> Dict[str, Dict[str, HpaOverrid
                     override_metadata = json.loads(override_json)
                     expire_after = override_metadata.get("expire_after")
                     min_instances = override_metadata.get("min_instances")
+                    max_instances = override_metadata.get("max_instances")
 
-                    if expire_after and min_instances:
+                    if expire_after and (min_instances or max_instances):
                         if current_time < expire_after:
                             if service not in overrides:
                                 overrides[service] = {}
 
                             overrides[service][instance] = {
                                 "min_instances": min_instances,
+                                "max_instances": max_instances,
                                 "expire_after": expire_after,
                             }
                             log.info(
                                 f"Found valid HPA override for {service}: "
-                                f"{override_metadata.get('min_instances')} (expires at {expire_after})"
+                                f"min_instances: {max_instances} :: max_instances {min_instances} (expires at {expire_after})"
                             )
                         else:
                             log.info(
@@ -320,7 +326,7 @@ def get_hpa_overrides(kube_client: KubeClient) -> Dict[str, Dict[str, HpaOverrid
                     else:
                         log.warning(
                             f"Invalid HPA override for {service}.{instance}: "
-                            f"missing 'min_instances' or 'expire_after': {override_metadata}"
+                            f"missing 'min_instances' or 'max_instances' or 'expire_after': {override_metadata}"
                         )
                 except Exception:
                     log.exception(

{paasta_tools-1.26.0 → paasta_tools-1.27.0/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: paasta-tools
-Version: 1.26.0
+Version: 1.27.0
 Summary: Tools for Yelps SOA infrastructure
 Author: Compute Infrastructure @ Yelp
 Author-email: compute-infra@yelp.com