paasta-tools 1.12.0__tar.gz → 1.13.0__tar.gz

{paasta-tools-1.12.0/paasta_tools.egg-info → paasta-tools-1.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.12.0
+Version: 1.13.0
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.12.0 → paasta-tools-1.13.0}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.12.0"
+__version__ = "1.13.0"

{paasta-tools-1.12.0 → paasta-tools-1.13.0}/paasta_tools/api/api.py

@@ -31,6 +31,7 @@ from wsgicors import CORS
 import paasta_tools.api
 from paasta_tools import kubernetes_tools
 from paasta_tools.api import settings
+from paasta_tools.api.tweens import auth
 from paasta_tools.api.tweens import profiling
 from paasta_tools.api.tweens import request_logger
 from paasta_tools.utils import load_system_paasta_config
@@ -79,6 +80,18 @@ def parse_paasta_api_args():
         default=4,
         help="Number of gunicorn workers to run",
     )
+    parser.add_argument(
+        "--auth-endpoint",
+        type=str,
+        default="",
+        help="External API authorization endpoint",
+    )
+    parser.add_argument(
+        "--auth-enforce",
+        action="store_true",
+        default=False,
+        help="Enforce API authorization",
+    )
     args = parser.parse_args()
     return args
 
@@ -105,6 +118,7 @@ def make_app(global_config=None):
 
     config.include("pyramid_swagger")
     config.include(request_logger)
+    config.include(auth)
 
     config.add_route(
         "flink.service.instance.jobs", "/v1/flink/{service}/{instance}/jobs"
@@ -257,6 +271,11 @@ def main(argv=None):
     if args.cluster:
         os.environ["PAASTA_API_CLUSTER"] = args.cluster
 
+    if args.auth_endpoint:
+        os.environ["PAASTA_API_AUTH_ENDPOINT"] = args.auth_endpoint
+        if args.auth_enforce:
+            os.environ["PAASTA_API_AUTH_ENFORCE"] = "1"
+
     gunicorn_args = [
         "gunicorn",
         "-w",

{paasta-tools-1.12.0 → paasta-tools-1.13.0}/paasta_tools/api/client.py

@@ -48,6 +48,7 @@ def get_paasta_oapi_client_by_url(
     cert_file: Optional[str] = None,
     key_file: Optional[str] = None,
     ssl_ca_cert: Optional[str] = None,
+    auth_token: str = "",
 ) -> PaastaOApiClient:
     server_variables = dict(scheme=parsed_url.scheme, host=parsed_url.netloc)
     config = paastaapi.Configuration(
@@ -63,6 +64,9 @@ def get_paasta_oapi_client_by_url(
     client.rest_client.pool_manager.connection_pool_kw[
         "timeout"
     ] = load_system_paasta_config().get_api_client_timeout()
+    # SEC-19555: support auth in PaaSTA APIs
+    if auth_token:
+        client.set_default_header("Authorization", f"Bearer {auth_token}")
     return PaastaOApiClient(
         autoscaler=paastaapis.AutoscalerApi(client),
         default=paastaapis.DefaultApi(client),
@@ -79,6 +83,7 @@ def get_paasta_oapi_client(
     cluster: str = None,
     system_paasta_config: SystemPaastaConfig = None,
     http_res: bool = False,
+    auth_token: str = "",
 ) -> Optional[PaastaOApiClient]:
     if not system_paasta_config:
         system_paasta_config = load_system_paasta_config()
@@ -94,4 +99,6 @@ def get_paasta_oapi_client(
     parsed = urlparse(api_endpoints[cluster])
     cert_file = key_file = ssl_ca_cert = None
 
-    return get_paasta_oapi_client_by_url(parsed, cert_file, key_file, ssl_ca_cert)
+    return get_paasta_oapi_client_by_url(
+        parsed, cert_file, key_file, ssl_ca_cert, auth_token
+    )

paasta-tools-1.13.0/paasta_tools/api/tweens/__init__.py

@@ -0,0 +1,6 @@
+from typing import Callable
+
+from pyramid.request import Request
+from pyramid.response import Response
+
+Handler = Callable[[Request], Response]

paasta-tools-1.13.0/paasta_tools/api/tweens/auth.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import json
+import logging
+import os
+from typing import NamedTuple
+from typing import Optional
+
+import cachetools.func
+import pyramid
+import requests
+from pyramid.config import Configurator
+from pyramid.httpexceptions import HTTPForbidden
+from pyramid.registry import Registry
+from pyramid.request import Request
+from pyramid.response import Response
+
+from paasta_tools.api.tweens import Handler
+
+
+logger = logging.getLogger(__name__)
+AUTH_CACHE_SIZE = 50000
+AUTH_CACHE_TTL = 30 * 60
+
+
+class AuthorizationOutcome(NamedTuple):
+    authorized: bool
+    reason: str
+
+
+class AuthTweenFactory:
+    def __init__(self, handler: Handler, registry: Registry) -> None:
+        self.handler = handler
+        self.registry = registry
+        self.enforce = bool(os.getenv("PAASTA_API_AUTH_ENFORCE", ""))
+        self.endpoint = os.getenv("PAASTA_API_AUTH_ENDPOINT")
+        self.session = requests.Session()
+
+    def __call__(self, request: Request) -> Response:
+        """
+        Extracts relevant metadata from request, and checks if it is authorized
+        """
+        token = request.headers.get("Authorization", "").strip()
+        token = token.split()[-1] if token else ""  # removes "Bearer" prefix
+        auth_outcome = self.is_request_authorized(
+            request.path,
+            token,
+            request.method,
+            request.swagger_data.get("service", None),
+        )
+        if self.enforce and not auth_outcome.authorized:
+            return HTTPForbidden(
+                body=json.dumps({"reason": auth_outcome.reason}),
+                headers={"X-Auth-Failure-Reason": auth_outcome.reason},
+                content_type="application/json",
+                charset="utf-8",
+            )
+        return self.handler(request)
+
+    @cachetools.func.ttl_cache(maxsize=AUTH_CACHE_SIZE, ttl=AUTH_CACHE_TTL)
+    def is_request_authorized(
+        self,
+        path: str,
+        token: str,
+        method: str,
+        service: Optional[str],
+    ) -> AuthorizationOutcome:
+        """Check if API request is authorized
+
+        :param str path: API path
+        :param str token: authentication token
+        :param str method: http method
+        :return: auth outcome
+        """
+        try:
+            response = self.session.post(
+                url=self.endpoint,
+                json={
+                    "input": {
+                        "path": path,
+                        "backend": "paasta",
+                        "token": token,
+                        "method": method,
+                        "service": service,
+                    },
+                },
+                timeout=2,
+            ).json()
+        except Exception as e:
+            logger.exception(f"Issue communicating with auth endpoint: {e}")
+            return AuthorizationOutcome(False, "Auth backend error")
+
+        auth_result_allowed = response.get("result", {}).get("allowed")
+        if auth_result_allowed is None:
+            return AuthorizationOutcome(False, "Malformed auth response")
+
+        if not auth_result_allowed:
+            reason = response["result"].get("reason", "Denied")
+            return AuthorizationOutcome(False, reason)
+
+        reason = response["result"].get("reason", "Ok")
+        return AuthorizationOutcome(True, reason)
+
+
+def includeme(config: Configurator):
+    if os.getenv("PAASTA_API_AUTH_ENDPOINT"):
+        config.add_tween(
+            "paasta_tools.api.tweens.auth.AuthTweenFactory",
+            under=(
+                pyramid.tweens.INGRESS,
+                "paasta_tools.api.tweens.request_logger.request_logger_tween_factory",
+            ),
+        )

{paasta-tools-1.12.0 → paasta-tools-1.13.0}/paasta_tools/cli/schemas/autoscaling_schema.json

@@ -6,6 +6,7 @@
             "type": {
                 "enum": [
                     "uwsgi",
+                    "uwsgi-v2",
                     "cpu",
                     "piscina",
                     "gunicorn",

{paasta-tools-1.12.0 → paasta-tools-1.13.0}/paasta_tools/kubernetes_tools.py

@@ -148,6 +148,7 @@ from paasta_tools.long_running_service_tools import METRICS_PROVIDER_GUNICORN
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_PISCINA
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_PROMQL
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_UWSGI
+from paasta_tools.long_running_service_tools import METRICS_PROVIDER_UWSGI_V2
 from paasta_tools.long_running_service_tools import ServiceNamespaceConfig
 from paasta_tools.secret_tools import get_secret_name_from_ref
 from paasta_tools.secret_tools import is_secret_ref
@@ -857,6 +858,20 @@ class KubernetesDeploymentConfig(LongRunningServiceConfig):
                     ),
                 ),
             )
+        elif provider["type"] == METRICS_PROVIDER_UWSGI_V2:
+            return V2MetricSpec(
+                type="Object",
+                object=V2ObjectMetricSource(
+                    metric=V2MetricIdentifier(name=prometheus_hpa_metric_name),
+                    described_object=V2CrossVersionObjectReference(
+                        api_version="apps/v1", kind="Deployment", name=name
+                    ),
+                    target=V2MetricTarget(
+                        type="AverageValue",
+                        average_value=target,
+                    ),
+                ),
+            )
 
         log.error(
             f"Unknown metrics_provider specified: {provider['type']} for\

{paasta-tools-1.12.0 → paasta-tools-1.13.0}/paasta_tools/long_running_service_tools.py

@@ -44,6 +44,7 @@ DEFAULT_GUNICORN_AUTOSCALING_MOVING_AVERAGE_WINDOW = 1800
 
 METRICS_PROVIDER_CPU = "cpu"
 METRICS_PROVIDER_UWSGI = "uwsgi"
+METRICS_PROVIDER_UWSGI_V2 = "uwsgi-v2"
 METRICS_PROVIDER_GUNICORN = "gunicorn"
 METRICS_PROVIDER_PISCINA = "piscina"
 METRICS_PROVIDER_ACTIVE_REQUESTS = "active-requests"
@@ -52,6 +53,7 @@ METRICS_PROVIDER_PROMQL = "arbitrary_promql"
 ALL_METRICS_PROVIDERS = [
     METRICS_PROVIDER_CPU,
     METRICS_PROVIDER_UWSGI,
+    METRICS_PROVIDER_UWSGI_V2,
     METRICS_PROVIDER_GUNICORN,
     METRICS_PROVIDER_PISCINA,
     METRICS_PROVIDER_ACTIVE_REQUESTS,

{paasta-tools-1.12.0 → paasta-tools-1.13.0}/paasta_tools/setup_prometheus_adapter_config.py

@@ -59,6 +59,7 @@ from paasta_tools.long_running_service_tools import METRICS_PROVIDER_GUNICORN
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_PISCINA
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_PROMQL
 from paasta_tools.long_running_service_tools import METRICS_PROVIDER_UWSGI
+from paasta_tools.long_running_service_tools import METRICS_PROVIDER_UWSGI_V2
 from paasta_tools.paasta_service_config_loader import PaastaServiceConfigLoader
 from paasta_tools.utils import DEFAULT_SOA_DIR
 from paasta_tools.utils import get_services_for_cluster
@@ -209,6 +210,10 @@ def create_instance_scaling_rule(
         return create_instance_uwsgi_scaling_rule(
             service, instance_config, metrics_provider_config, paasta_cluster
         )
+    if metrics_provider_config["type"] == METRICS_PROVIDER_UWSGI_V2:
+        return create_instance_uwsgi_v2_scaling_rule(
+            service, instance_config, metrics_provider_config, paasta_cluster
+        )
     if metrics_provider_config["type"] == METRICS_PROVIDER_PISCINA:
         return create_instance_piscina_scaling_rule(
             service, instance_config, metrics_provider_config, paasta_cluster
@@ -445,6 +450,79 @@ def create_instance_uwsgi_scaling_rule(
     }
 
 
+def create_instance_uwsgi_v2_scaling_rule(
+    service: str,
+    instance_config: KubernetesDeploymentConfig,
+    metrics_provider_config: MetricsProviderDict,
+    paasta_cluster: str,
+) -> PrometheusAdapterRule:
+    """
+    Creates a Prometheus adapter rule config for a given service instance.
+    """
+    instance = instance_config.instance
+    moving_average_window = metrics_provider_config.get(
+        "moving_average_window_seconds", DEFAULT_UWSGI_AUTOSCALING_MOVING_AVERAGE_WINDOW
+    )
+    deployment_name = get_kubernetes_app_name(service=service, instance=instance)
+
+    # In order for autoscaling to work safely while a service migrates from one namespace to another, the HPA needs to
+    # make sure that the deployment in the new namespace is scaled up enough to handle _all_ the load.
+    # This is because once the new deployment is 100% healthy, cleanup_kubernetes_job will delete the deployment out of
+    # the old namespace all at once, suddenly putting all the load onto the deployment in the new namespace.
+    # To ensure this, we must NOT filter on namespace in worker_filter_terms (which is used when calculating total_load.
+    # This makes sure that desired_instances includes load from all namespaces.
+    worker_filter_terms = f"paasta_cluster='{paasta_cluster}',paasta_service='{service}',paasta_instance='{instance}'"
+
+    # k8s:deployment:pods_status_ready is a metric created by summing kube_pod_status_ready
+    # over paasta service/instance/cluster. it counts the number of ready pods in a paasta
+    # deployment.
+    ready_pods = f"""
+        (sum(
+            k8s:deployment:pods_status_ready{{{worker_filter_terms}}} >= 0
+            or
+            max_over_time(
+                k8s:deployment:pods_status_ready{{{worker_filter_terms}}}[{DEFAULT_EXTRAPOLATION_TIME}s]
+            )
+        ) by (kube_deployment))
+    """
+    load_per_instance = f"""
+        avg(
+            uwsgi_worker_busy{{{worker_filter_terms}}}
+        ) by (kube_pod, kube_deployment)
+    """
+    missing_instances = f"""
+        clamp_min(
+            {ready_pods} - count({load_per_instance}) by (kube_deployment),
+            0
+        )
+    """
+    total_load = f"""
+    (
+        sum(
+            {load_per_instance}
+        ) by (kube_deployment)
+        +
+        {missing_instances}
+    )
+    """
+    total_load_smoothed = f"""
+        avg_over_time(
+            (
+                {total_load}
+            )[{moving_average_window}s:]
+        )
+    """
+
+    metric_name = f"{deployment_name}-uwsgi-v2-prom"
+
+    return {
+        "name": {"as": metric_name},
+        "seriesQuery": f"uwsgi_worker_busy{{{worker_filter_terms}}}",
+        "resources": {"template": "kube_<<.Resource>>"},
+        "metricsQuery": _minify_promql(total_load_smoothed),
+    }
+
+
 def create_instance_piscina_scaling_rule(
     service: str,
     instance_config: KubernetesDeploymentConfig,

{paasta-tools-1.12.0 → paasta-tools-1.13.0/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.12.0
+Version: 1.13.0
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.12.0 → paasta-tools-1.13.0}/paasta_tools.egg-info/SOURCES.txt

@@ -99,6 +99,7 @@ paasta_tools/api/client.py
 paasta_tools/api/settings.py
 paasta_tools/api/api_docs/swagger.json
 paasta_tools/api/tweens/__init__.py
+paasta_tools/api/tweens/auth.py
 paasta_tools/api/tweens/profiling.py
 paasta_tools/api/tweens/request_logger.py
 paasta_tools/api/views/__init__.py