paasta-tools 1.12.1__tar.gz → 1.13.1__tar.gz

{paasta-tools-1.12.1/paasta_tools.egg-info → paasta-tools-1.13.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.12.1
+Version: 1.13.1
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.12.1 → paasta-tools-1.13.1}/paasta_tools/__init__.py

@@ -17,4 +17,4 @@
 # setup phase, the dependencies may not exist on disk yet.
 #
 # Don't bump version manually. See `make release` docs in ./Makefile
-__version__ = "1.12.1"
+__version__ = "1.13.1"

{paasta-tools-1.12.1 → paasta-tools-1.13.1}/paasta_tools/api/api.py

@@ -24,13 +24,14 @@ import sys
 import manhole
 import requests_cache
 import service_configuration_lib
-import yaml
 from pyramid.config import Configurator
 from wsgicors import CORS
 
 import paasta_tools.api
 from paasta_tools import kubernetes_tools
+from paasta_tools import yaml_tools as yaml
 from paasta_tools.api import settings
+from paasta_tools.api.tweens import auth
 from paasta_tools.api.tweens import profiling
 from paasta_tools.api.tweens import request_logger
 from paasta_tools.utils import load_system_paasta_config
@@ -79,6 +80,18 @@ def parse_paasta_api_args():
         default=4,
         help="Number of gunicorn workers to run",
     )
+    parser.add_argument(
+        "--auth-endpoint",
+        type=str,
+        default="",
+        help="External API authorization endpoint",
+    )
+    parser.add_argument(
+        "--auth-enforce",
+        action="store_true",
+        default=False,
+        help="Enforce API authorization",
+    )
     args = parser.parse_args()
     return args
 
@@ -105,6 +118,7 @@ def make_app(global_config=None):
 
     config.include("pyramid_swagger")
     config.include(request_logger)
+    config.include(auth)
 
     config.add_route(
         "flink.service.instance.jobs", "/v1/flink/{service}/{instance}/jobs"
@@ -257,6 +271,11 @@ def main(argv=None):
     if args.cluster:
         os.environ["PAASTA_API_CLUSTER"] = args.cluster
 
+    if args.auth_endpoint:
+        os.environ["PAASTA_API_AUTH_ENDPOINT"] = args.auth_endpoint
+        if args.auth_enforce:
+            os.environ["PAASTA_API_AUTH_ENFORCE"] = "1"
+
     gunicorn_args = [
         "gunicorn",
         "-w",

{paasta-tools-1.12.1 → paasta-tools-1.13.1}/paasta_tools/api/client.py

@@ -48,6 +48,7 @@ def get_paasta_oapi_client_by_url(
     cert_file: Optional[str] = None,
     key_file: Optional[str] = None,
     ssl_ca_cert: Optional[str] = None,
+    auth_token: str = "",
 ) -> PaastaOApiClient:
     server_variables = dict(scheme=parsed_url.scheme, host=parsed_url.netloc)
     config = paastaapi.Configuration(
@@ -63,6 +64,9 @@ def get_paasta_oapi_client_by_url(
     client.rest_client.pool_manager.connection_pool_kw[
         "timeout"
     ] = load_system_paasta_config().get_api_client_timeout()
+    # SEC-19555: support auth in PaaSTA APIs
+    if auth_token:
+        client.set_default_header("Authorization", f"Bearer {auth_token}")
     return PaastaOApiClient(
         autoscaler=paastaapis.AutoscalerApi(client),
         default=paastaapis.DefaultApi(client),
@@ -79,6 +83,7 @@ def get_paasta_oapi_client(
     cluster: str = None,
     system_paasta_config: SystemPaastaConfig = None,
     http_res: bool = False,
+    auth_token: str = "",
 ) -> Optional[PaastaOApiClient]:
     if not system_paasta_config:
         system_paasta_config = load_system_paasta_config()
@@ -94,4 +99,6 @@ def get_paasta_oapi_client(
     parsed = urlparse(api_endpoints[cluster])
     cert_file = key_file = ssl_ca_cert = None
 
-    return get_paasta_oapi_client_by_url(parsed, cert_file, key_file, ssl_ca_cert)
+    return get_paasta_oapi_client_by_url(
+        parsed, cert_file, key_file, ssl_ca_cert, auth_token
+    )

paasta-tools-1.13.1/paasta_tools/api/tweens/__init__.py

@@ -0,0 +1,6 @@
+from typing import Callable
+
+from pyramid.request import Request
+from pyramid.response import Response
+
+Handler = Callable[[Request], Response]

paasta-tools-1.13.1/paasta_tools/api/tweens/auth.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import json
+import logging
+import os
+from typing import NamedTuple
+from typing import Optional
+
+import cachetools.func
+import pyramid
+import requests
+from pyramid.config import Configurator
+from pyramid.httpexceptions import HTTPForbidden
+from pyramid.registry import Registry
+from pyramid.request import Request
+from pyramid.response import Response
+
+from paasta_tools.api.tweens import Handler
+
+
+logger = logging.getLogger(__name__)
+AUTH_CACHE_SIZE = 50000
+AUTH_CACHE_TTL = 30 * 60
+
+
+class AuthorizationOutcome(NamedTuple):
+    authorized: bool
+    reason: str
+
+
+class AuthTweenFactory:
+    def __init__(self, handler: Handler, registry: Registry) -> None:
+        self.handler = handler
+        self.registry = registry
+        self.enforce = bool(os.getenv("PAASTA_API_AUTH_ENFORCE", ""))
+        self.endpoint = os.getenv("PAASTA_API_AUTH_ENDPOINT")
+        self.session = requests.Session()
+
+    def __call__(self, request: Request) -> Response:
+        """
+        Extracts relevant metadata from request, and checks if it is authorized
+        """
+        token = request.headers.get("Authorization", "").strip()
+        token = token.split()[-1] if token else ""  # removes "Bearer" prefix
+        auth_outcome = self.is_request_authorized(
+            request.path,
+            token,
+            request.method,
+            request.swagger_data.get("service", None),
+        )
+        if self.enforce and not auth_outcome.authorized:
+            return HTTPForbidden(
+                body=json.dumps({"reason": auth_outcome.reason}),
+                headers={"X-Auth-Failure-Reason": auth_outcome.reason},
+                content_type="application/json",
+                charset="utf-8",
+            )
+        return self.handler(request)
+
+    @cachetools.func.ttl_cache(maxsize=AUTH_CACHE_SIZE, ttl=AUTH_CACHE_TTL)
+    def is_request_authorized(
+        self,
+        path: str,
+        token: str,
+        method: str,
+        service: Optional[str],
+    ) -> AuthorizationOutcome:
+        """Check if API request is authorized
+
+        :param str path: API path
+        :param str token: authentication token
+        :param str method: http method
+        :return: auth outcome
+        """
+        try:
+            response = self.session.post(
+                url=self.endpoint,
+                json={
+                    "input": {
+                        "path": path,
+                        "backend": "paasta",
+                        "token": token,
+                        "method": method,
+                        "service": service,
+                    },
+                },
+                timeout=2,
+            ).json()
+        except Exception as e:
+            logger.exception(f"Issue communicating with auth endpoint: {e}")
+            return AuthorizationOutcome(False, "Auth backend error")
+
+        auth_result_allowed = response.get("result", {}).get("allowed")
+        if auth_result_allowed is None:
+            return AuthorizationOutcome(False, "Malformed auth response")
+
+        if not auth_result_allowed:
+            reason = response["result"].get("reason", "Denied")
+            return AuthorizationOutcome(False, reason)
+
+        reason = response["result"].get("reason", "Ok")
+        return AuthorizationOutcome(True, reason)
+
+
+def includeme(config: Configurator):
+    if os.getenv("PAASTA_API_AUTH_ENDPOINT"):
+        config.add_tween(
+            "paasta_tools.api.tweens.auth.AuthTweenFactory",
+            under=(
+                pyramid.tweens.INGRESS,
+                "paasta_tools.api.tweens.request_logger.request_logger_tween_factory",
+            ),
+        )

{paasta-tools-1.12.1 → paasta-tools-1.13.1}/paasta_tools/cli/cmds/validate.py

@@ -33,7 +33,6 @@ from typing import Tuple
 from typing import Union
 
 import pytz
-import yaml
 from croniter import croniter
 from jsonschema import Draft4Validator
 from jsonschema import exceptions
@@ -45,6 +44,7 @@ from ruamel.yaml import SafeConstructor
 from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap
 
+from paasta_tools import yaml_tools as yaml
 from paasta_tools.autoscaling.utils import MetricsProviderDict
 from paasta_tools.cli.utils import failure
 from paasta_tools.cli.utils import get_file_contents

{paasta-tools-1.12.1 → paasta-tools-1.13.1}/paasta_tools/cli/fsm/autosuggest.py

@@ -14,7 +14,7 @@
 import os
 import random
 
-import yaml
+from paasta_tools import yaml_tools as yaml
 
 
 def _get_smartstack_proxy_ports_from_file(root, file):

{paasta-tools-1.12.1 → paasta-tools-1.13.1}/paasta_tools/contrib/check_orphans.py

@@ -19,10 +19,11 @@ from typing import Optional
 from typing import Set
 from typing import Tuple
 
-import yaml
 from kazoo.client import KazooClient
 from kazoo.exceptions import NoNodeError
 
+from paasta_tools import yaml_tools as yaml
+
 logger = logging.getLogger("check_orphans")
 
 PREFIX = "/smartstack/global/"

{paasta-tools-1.12.1 → paasta-tools-1.13.1}/paasta_tools/contrib/render_template.py

@@ -3,7 +3,7 @@ import argparse
 import os
 import re
 
-import yaml
+from paasta_tools import yaml_tools as yaml
 
 
 def replace(s, values):

{paasta-tools-1.12.1 → paasta-tools-1.13.1}/paasta_tools/contrib/timeouts_metrics_prom.py

@@ -16,12 +16,12 @@ yelpsoaconfig_endpoint_timeouts_ms{path="/",upstream="mysql_read_security.main.e
 """
 import os
 
-import yaml
 from prometheus_client import CollectorRegistry
 from prometheus_client import Gauge
 from prometheus_client import write_to_textfile
 from prometheus_client.metrics import MetricWrapperBase
 
+from paasta_tools import yaml_tools as yaml
 from paasta_tools.utils import DEFAULT_SOA_DIR
 
 PROM_OUTPUT_FILE = f"{DEFAULT_SOA_DIR}/.autotune_timeouts.prom"

{paasta-tools-1.12.1 → paasta-tools-1.13.1}/paasta_tools/envoy_tools.py

@@ -30,10 +30,10 @@ from typing import Set
 from typing import Tuple
 
 import requests
-import yaml
 from kubernetes.client import V1Pod
 from mypy_extensions import TypedDict
 
+from paasta_tools import yaml_tools as yaml
 from paasta_tools.utils import get_user_agent
 
 

{paasta-tools-1.12.1 → paasta-tools-1.13.1}/paasta_tools/generate_authenticating_services.py

@@ -24,8 +24,7 @@ from typing import Dict
 from typing import List
 from typing import Set
 
-import yaml
-
+from paasta_tools import yaml_tools as yaml
 from paasta_tools.utils import DEFAULT_SOA_DIR
 from paasta_tools.utils import load_system_paasta_config
 from paasta_tools.utils import write_json_configuration_file

{paasta-tools-1.12.1 → paasta-tools-1.13.1}/paasta_tools/setup_istio_mesh.py

@@ -32,8 +32,8 @@ from typing import Mapping
 from typing import Set
 
 import kubernetes.client as k8s
-import yaml
 
+from paasta_tools import yaml_tools as yaml
 from paasta_tools.kubernetes_tools import ensure_namespace
 from paasta_tools.kubernetes_tools import KubeClient
 from paasta_tools.kubernetes_tools import limit_size_with_hash
@@ -99,7 +99,7 @@ def load_smartstack_namespaces(soa_dir: str = DEFAULT_SOA_DIR) -> Mapping:
             continue
         try:
             with open(file_path) as f:
-                svc_namespaces = yaml.load(f, Loader=yaml.CSafeLoader)
+                svc_namespaces = yaml.safe_load(f)
                 for (ns, details) in svc_namespaces.items():
                     namespaces[f"{dir}.{ns}"] = details
         except Exception as err:

{paasta-tools-1.12.1 → paasta-tools-1.13.1}/paasta_tools/setup_kubernetes_cr.py

@@ -28,9 +28,9 @@ from typing import Mapping
 from typing import Optional
 from typing import Sequence
 
-import yaml
 from kubernetes.client.exceptions import ApiException
 
+from paasta_tools import yaml_tools as yaml
 from paasta_tools.cli.utils import LONG_RUNNING_INSTANCE_TYPE_HANDLERS
 from paasta_tools.flink_tools import get_flink_ingress_url_root
 from paasta_tools.kubernetes_tools import create_custom_resource

{paasta-tools-1.12.1 → paasta-tools-1.13.1}/paasta_tools/tron/client.py

@@ -15,8 +15,8 @@ from typing import Dict
 from urllib.parse import urljoin
 
 import requests
-import yaml
 
+from paasta_tools import yaml_tools as yaml
 from paasta_tools.utils import get_user_agent
 
 

{paasta-tools-1.12.1 → paasta-tools-1.13.1}/paasta_tools/tron_tools.py

@@ -25,13 +25,13 @@ from typing import Mapping
 from typing import Tuple
 from typing import Union
 
-import yaml
 from mypy_extensions import TypedDict
 from service_configuration_lib import read_extra_service_information
 from service_configuration_lib import read_yaml_file
 from service_configuration_lib.spark_config import get_total_driver_memory_mb
 from service_configuration_lib.spark_config import SparkConfBuilder
 
+from paasta_tools import yaml_tools as yaml
 from paasta_tools.mesos_tools import mesos_services_running_here
 
 try:

{paasta-tools-1.12.1 → paasta-tools-1.13.1}/paasta_tools/utils.py

@@ -74,7 +74,6 @@ import dateutil.tz
 import ldap3
 import requests_cache
 import service_configuration_lib
-import yaml
 from docker import APIClient
 from docker.utils import kwargs_from_env
 from kazoo.client import KazooClient
@@ -83,6 +82,7 @@ from service_configuration_lib import read_extra_service_information
 from service_configuration_lib import read_service_configuration
 
 import paasta_tools.cli.fsm
+from paasta_tools import yaml_tools as yaml
 
 
 # DO NOT CHANGE SPACER, UNLESS YOU'RE PREPARED TO CHANGE ALL INSTANCES

paasta-tools-1.13.1/paasta_tools/yaml_tools.py

@@ -0,0 +1,44 @@
+import sys
+
+import yaml
+
+# try and catch both /opt/venvs/paasta-tools and ~/pg/paasta/.tox/py38-linux as if we're being run as an application,
+# we likely want to fail on a potential slowdown rather than experience a performance regression
+if "paasta" in sys.prefix:
+    from yaml import CSafeLoader as Loader
+    from yaml import CSafeDumper as Dumper
+# but for the vanishingly few instances where folks add us as as dependency, we don't enforce that they use the
+# C-accelerated Loader/Dumper
+else:
+    try:
+        from yaml import CSafeLoader as Loader
+        from yaml import CSafeDumper as Dumper
+    except ImportError:  # pragma: no cover
+        from yaml import SafeLoader as Loader  # type: ignore
+        from yaml import SafeDumper as Dumper  # type: ignore
+
+
+def dump(*args, **kwargs):
+    kwargs["Dumper"] = Dumper
+    return yaml.dump(*args, **kwargs)
+
+
+def dump_all(*args, **kwargs):
+    kwargs["Dumper"] = Dumper
+    return yaml.dump_all(*args, **kwargs)
+
+
+def load(*args, **kwargs):
+    kwargs["Loader"] = Loader
+    return yaml.load(*args, **kwargs)
+
+
+def load_all(*args, **kwargs):
+    kwargs["Loader"] = Loader
+    return yaml.load_all(*args, **kwargs)
+
+
+safe_dump = dump
+safe_dump_all = dump_all
+safe_load = load
+safe_load_all = load_all

{paasta-tools-1.12.1 → paasta-tools-1.13.1/paasta_tools.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: paasta-tools
-Version: 1.12.1
+Version: 1.13.1
 Summary: Tools for Yelps SOA infrastructure
 Home-page: UNKNOWN
 Author: Compute Infrastructure @ Yelp

{paasta-tools-1.12.1 → paasta-tools-1.13.1}/paasta_tools.egg-info/SOURCES.txt

@@ -87,6 +87,7 @@ paasta_tools/spark_tools.py
 paasta_tools/synapse_srv_namespaces_fact.py
 paasta_tools/tron_tools.py
 paasta_tools/utils.py
+paasta_tools/yaml_tools.py
 paasta_tools.egg-info/PKG-INFO
 paasta_tools.egg-info/SOURCES.txt
 paasta_tools.egg-info/dependency_links.txt
@@ -99,6 +100,7 @@ paasta_tools/api/client.py
 paasta_tools/api/settings.py
 paasta_tools/api/api_docs/swagger.json
 paasta_tools/api/tweens/__init__.py
+paasta_tools/api/tweens/auth.py
 paasta_tools/api/tweens/profiling.py
 paasta_tools/api/tweens/request_logger.py
 paasta_tools/api/views/__init__.py