paasta-tools 1.21.3__py3-none-any.whl

paasta_tools/api/views/resources.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+PaaSTA resource utilization, etc.
+"""
+from a_sync import block
+from pyramid.response import Response
+from pyramid.view import view_config
+
+from paasta_tools.mesos_tools import get_mesos_master
+from paasta_tools.metrics import metastatus_lib
+
+
+def parse_filters(filters):
+    # The swagger config verifies that the data is in this format
+    #  "pattern": "(.*):(.*,)*(.*)"
+    if filters is None:
+        return {}
+    f = {s[0]: s[1] for s in [e.split(":") for e in filters]}
+    f = {k: v.split(",") for k, v in f.items()}
+    return f
+
+
+@view_config(route_name="resources.utilization", request_method="GET", renderer="json")
+def resources_utilization(request):
+    master = get_mesos_master()
+    mesos_state = block(master.state)
+
+    groupings = request.swagger_data.get("groupings", ["superregion"])
+    # swagger actually makes the key None if it's not set
+    if groupings is None:
+        groupings = ["superregion"]
+    grouping_function = metastatus_lib.key_func_for_attribute_multi(groupings)
+    sorting_function = metastatus_lib.sort_func_for_attributes(groupings)
+
+    filters = request.swagger_data.get("filter", [])
+    filters = parse_filters(filters)
+    filter_funcs = [
+        metastatus_lib.make_filter_slave_func(attr, vals)
+        for attr, vals in filters.items()
+    ]
+
+    resource_info_dict = metastatus_lib.get_resource_utilization_by_grouping(
+        grouping_func=grouping_function,
+        mesos_state=mesos_state,
+        filters=filter_funcs,
+        sort_func=sorting_function,
+    )
+
+    response_body = []
+    for k, v in resource_info_dict.items():
+        group = {"groupings": {}}
+        for grouping, value in k:
+            group["groupings"][grouping] = value
+        for resource, value in v["total"]._asdict().items():
+            group[resource] = {"total": value}
+        for resource, value in v["free"]._asdict().items():
+            group[resource]["free"] = value
+        for resource in v["free"]._fields:
+            group[resource]["used"] = group[resource]["total"] - group[resource]["free"]
+
+        response_body.append(group)
+
+    return Response(json_body=response_body, status_code=200)

paasta_tools/api/views/service.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+PaaSTA service list (instances) etc.
+"""
+from pyramid.view import view_config
+
+from paasta_tools.api import settings
+from paasta_tools.utils import get_services_for_cluster
+from paasta_tools.utils import list_all_instances_for_service
+
+
+@view_config(route_name="service.list", request_method="GET", renderer="json")
+def list_instances(request):
+    service = request.swagger_data.get("service")
+    instances = list_all_instances_for_service(service, clusters=[settings.cluster])
+    return {"instances": list(instances)}
+
+
+@view_config(route_name="services", request_method="GET", renderer="json")
+def list_services_for_cluster(request):
+    services_for_cluster = get_services_for_cluster(cluster=settings.cluster)
+    return {"services": services_for_cluster}

paasta_tools/api/views/version.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+PaaSTA service list (instances) etc.
+"""
+from pyramid.view import view_config
+
+from paasta_tools import __version__
+
+
+@view_config(route_name="version", request_method="GET", renderer="json")
+def version(request):
+    return __version__

paasta_tools/apply_external_resources.py

@@ -0,0 +1,79 @@
+#!/opt/venvs/paasta-tools/bin/python
+import os
+import sys
+from filecmp import cmp
+from shutil import copy
+from subprocess import CalledProcessError
+from subprocess import run
+from traceback import print_exc
+
+APPLIED_DIRECTORY = ".applied"
+
+
+# This script expects the KUBECONFIG environment variable to be set correctly
+def main(puppet_resource_root: str) -> int:
+    exit_code = 0
+    applied_resource_root = os.path.join(puppet_resource_root, APPLIED_DIRECTORY)
+
+    # Loop through everything in the puppet resource path
+    for root, dirs, files in os.walk(puppet_resource_root):
+        # modifying the 'dirs' variable in-place will update the order that os.walk visits things
+        if APPLIED_DIRECTORY in dirs:
+            dirs.remove(APPLIED_DIRECTORY)
+        dirs.sort()  # Need to apply things in the correct order
+
+        # Check to see if there's a difference between what Puppet created and
+        # what's been previously applied
+        for filename in sorted([f for f in files if f.endswith(".yaml")]):
+            path = os.path.join(root, filename)
+            applied_path = os.path.join(
+                applied_resource_root, os.path.relpath(path, puppet_resource_root)
+            )
+            print(f"comparing {path} and {applied_path}")
+            if not os.path.exists(applied_path) or not cmp(
+                path, applied_path, shallow=False
+            ):
+                # This is idempotent; if something gets out of sync and a resource gets applied
+                # a second time, kubectl just won't make any changes
+                try:
+                    run(["kubectl", "apply", "-f", path], check=True)
+                    os.makedirs(os.path.dirname(applied_path), exist_ok=True)
+                    copy(path, applied_path)
+                except CalledProcessError:
+                    print(f"There was a problem applying {path}:\n")
+                    print_exc(
+                        file=sys.stdout
+                    )  # keep all messages on the same stream so they're in order
+                    exit_code = 1
+                    continue
+
+    # Loop through all the files that have been previously applied and see
+    # if Puppet has removed any of them
+    for root, dirs, files in os.walk(applied_resource_root):
+        dirs.sort(reverse=True)  # for deleting things, we need to go in reverse order
+        for filename in sorted([f for f in files if f.endswith(".yaml")], reverse=True):
+            path = os.path.join(root, filename)
+            puppet_path = os.path.join(
+                puppet_resource_root, os.path.relpath(path, applied_resource_root)
+            )
+            if not os.path.exists(puppet_path):
+                print(f"Deleting resource {path}...")
+                try:
+                    run(
+                        ["kubectl", "delete", "--ignore-not-found=true", "-f", path],
+                        check=True,
+                    )
+                    os.remove(path)
+                except CalledProcessError:
+                    print(f"There was a problem deleting {path}:\n")
+                    print_exc(
+                        file=sys.stdout
+                    )  # keep all messages on the same stream so they're in order
+                    exit_code = 1
+                    continue
+
+    return exit_code
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1]))

paasta_tools/async_utils.py

@@ -0,0 +1,109 @@
+import asyncio
+import functools
+import time
+import weakref
+from collections import defaultdict
+from typing import AsyncIterable
+from typing import Awaitable
+from typing import Callable
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import TypeVar
+
+
+T = TypeVar("T")
+
+
+# NOTE: this method is not thread-safe due to lack of locking while checking
+# and updating the cache
+def async_ttl_cache(
+    ttl: Optional[float] = 300,
+    cleanup_self: bool = False,
+    *,
+    cache: Optional[Dict] = None,
+) -> Callable[
+    [Callable[..., Awaitable[T]]], Callable[..., Awaitable[T]]  # wrapped  # inner
+]:
+    async def call_or_get_from_cache(cache, async_func, args_for_key, args, kwargs):
+        # Please note that anything which is put into `key` will be in the
+        # cache forever, potentially causing memory leaks.  The most common
+        # case is the `self` arg pointing to a huge object.  To mitigate that
+        # we're using `args_for_key`, which is supposed not contain any huge
+        # objects.
+        key = functools._make_key(args_for_key, kwargs, typed=False)
+        try:
+            future, last_update = cache[key]
+            if ttl is not None and time.time() - last_update > ttl:
+                raise KeyError
+        except KeyError:
+            future = asyncio.ensure_future(async_func(*args, **kwargs))
+            # set the timestamp to +infinity so that we always wait on the in-flight request.
+            cache[key] = (future, float("Inf"))
+
+        try:
+            value = await future
+        except Exception:
+            # Only update the cache if it's the same future we awaited and
+            # it hasn't already been updated by another coroutine
+            # Note also that we use get() in case the key was deleted from the
+            # cache by another coroutine
+            if cache.get(key) == (future, float("Inf")):
+                del cache[key]
+            raise
+        else:
+            if cache.get(key) == (future, float("Inf")):
+                cache[key] = (future, time.time())
+            return value
+
+    if cleanup_self:
+        instance_caches: Dict = cache if cache is not None else defaultdict(dict)
+
+        def on_delete(w):
+            del instance_caches[w]
+
+        def outer(wrapped):
+            @functools.wraps(wrapped)
+            async def inner(self, *args, **kwargs):
+                w = weakref.ref(self, on_delete)
+                self_cache = instance_caches[w]
+                return await call_or_get_from_cache(
+                    self_cache, wrapped, args, (self,) + args, kwargs
+                )
+
+            return inner
+
+    else:
+        cache2: Dict = (
+            cache if cache is not None else {}
+        )  # Should be Dict[Any, T] but that doesn't work.
+
+        def outer(wrapped):
+            @functools.wraps(wrapped)
+            async def inner(*args, **kwargs):
+                return await call_or_get_from_cache(cache2, wrapped, args, args, kwargs)
+
+            return inner
+
+    return outer
+
+
+async def aiter_to_list(
+    aiter: AsyncIterable[T],
+) -> List[T]:
+    return [x async for x in aiter]
+
+
+def async_timeout(
+    seconds: int = 10,
+) -> Callable[
+    [Callable[..., Awaitable[T]]], Callable[..., Awaitable[T]]  # wrapped  # inner
+]:
+    def outer(wrapped):
+        @functools.wraps(wrapped)
+        async def inner(*args, **kwargs):
+            return await asyncio.wait_for(wrapped(*args, **kwargs), timeout=seconds)
+
+        return inner
+
+    return outer

paasta_tools/autoscaling/autoscaling_service_lib.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# Copyright 2015-2019 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import time
+from collections import namedtuple
+
+from kazoo.exceptions import NoNodeError
+
+from paasta_tools.long_running_service_tools import ZK_PAUSE_AUTOSCALE_PATH
+from paasta_tools.utils import ZookeeperPool
+
+ServiceAutoscalingInfo = namedtuple(
+    "ServiceAutoscalingInfo",
+    [
+        "current_instances",
+        "max_instances",
+        "min_instances",
+        "current_utilization",
+        "target_instances",
+    ],
+)
+
+
+SERVICE_METRICS_PROVIDER_KEY = "metrics_providers"
+DECISION_POLICY_KEY = "decision_policy"
+
+
+log = logging.getLogger(__name__)
+log.addHandler(logging.NullHandler())
+
+
+def autoscaling_is_paused():
+    with ZookeeperPool() as zk:
+        try:
+            pause_until = zk.get(ZK_PAUSE_AUTOSCALE_PATH)[0].decode("utf8")
+            pause_until = float(pause_until)
+        except (NoNodeError, ValueError, AttributeError):
+            pause_until = 0
+
+    remaining = pause_until - time.time()
+    if remaining >= 0:
+        log.debug("Autoscaling is paused for {} more seconds".format(str(remaining)))
+        return True
+    else:
+        return False

paasta_tools/autoscaling/forecasting.py

@@ -0,0 +1,106 @@
+from paasta_tools.autoscaling.utils import get_autoscaling_component
+from paasta_tools.autoscaling.utils import register_autoscaling_component
+from paasta_tools.long_running_service_tools import (
+    DEFAULT_UWSGI_AUTOSCALING_MOVING_AVERAGE_WINDOW,
+)
+
+
+FORECAST_POLICY_KEY = "forecast_policy"
+
+
+def get_forecast_policy(name):
+    """
+    Returns a forecast policy matching the given name. Only used by decision policies that try to forecast load, like
+    the proportional decision policy.
+    """
+    return get_autoscaling_component(name, FORECAST_POLICY_KEY)
+
+
+@register_autoscaling_component("current", FORECAST_POLICY_KEY)
+def current_value_forecast_policy(historical_load, **kwargs):
+    """A prediction policy that assumes that the value any time in the future will be the same as the current value.
+
+    :param historical_load: a list of (timestamp, value)s, where timestamp is a unix timestamp and value is load.
+    """
+    return historical_load[-1][1]
+
+
+def window_historical_load(historical_load, window_begin, window_end):
+    """Filter historical_load down to just the datapoints lying between times window_begin and window_end, inclusive."""
+    filtered = []
+    for timestamp, value in historical_load:
+        if timestamp >= window_begin and timestamp <= window_end:
+            filtered.append((timestamp, value))
+    return filtered
+
+
+def trailing_window_historical_load(historical_load, window_size):
+    window_end, _ = historical_load[-1]
+    window_begin = window_end - window_size
+    return window_historical_load(historical_load, window_begin, window_end)
+
+
+@register_autoscaling_component("moving_average", FORECAST_POLICY_KEY)
+def moving_average_forecast_policy(
+    historical_load,
+    moving_average_window_seconds=DEFAULT_UWSGI_AUTOSCALING_MOVING_AVERAGE_WINDOW,
+    **kwargs,
+):
+    """Does a simple average of all historical load data points within the moving average window. Weights all data
+    points within the window equally."""
+
+    windowed_data = trailing_window_historical_load(
+        historical_load, moving_average_window_seconds
+    )
+    windowed_values = [value for timestamp, value in windowed_data]
+    return sum(windowed_values) / len(windowed_values)
+
+
+@register_autoscaling_component("linreg", FORECAST_POLICY_KEY)
+def linreg_forecast_policy(
+    historical_load,
+    linreg_window_seconds,
+    linreg_extrapolation_seconds,
+    linreg_default_slope=0,
+    **kwargs,
+):
+    """Does a linear regression on the load data within the last linreg_window_seconds. For every time delta in
+    linreg_extrapolation_seconds, forecasts the value at that time delta from now, and returns the maximum of these
+    predicted values. (With linear extrapolation, it doesn't make sense to forecast at more than two points, as the max
+    load will always be at the first or last time delta.)
+
+    :param linreg_window_seconds: Consider all data from this many seconds ago until now.
+    :param linreg_extrapolation_seconds: A list of floats representing a number of seconds in the future at which to
+                                         predict the load. The highest prediction will be returned.
+    :param linreg_default_slope: If there is only one data point within the window, the equation for slope is undefined,
+                                 so we use this value (expressed in load/second) for prediction instead. Default is
+                                 0.
+
+    """
+
+    window = trailing_window_historical_load(historical_load, linreg_window_seconds)
+
+    loads = [load for timestamp, load in window]
+    times = [timestamp for timestamp, load in window]
+
+    mean_time = sum(times) / len(times)
+    mean_load = sum(loads) / len(loads)
+
+    if len(window) > 1:
+        slope = sum((t - mean_time) * (l - mean_load) for t, l in window) / sum(
+            (t - mean_time) ** 2 for t in times
+        )
+    else:
+        slope = linreg_default_slope
+
+    intercept = mean_load - slope * mean_time
+
+    def predict(timestamp):
+        return slope * timestamp + intercept
+
+    if isinstance(linreg_extrapolation_seconds, (int, float)):
+        linreg_extrapolation_seconds = [linreg_extrapolation_seconds]
+
+    now, _ = historical_load[-1]
+    forecasted_values = [predict(now + delta) for delta in linreg_extrapolation_seconds]
+    return max(forecasted_values)

paasta_tools/autoscaling/max_all_k8s_services.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+from paasta_tools.kubernetes.application.controller_wrappers import (
+    get_application_wrapper,
+)
+from paasta_tools.kubernetes_tools import KubeClient
+from paasta_tools.kubernetes_tools import KubernetesDeploymentConfig
+from paasta_tools.paasta_service_config_loader import PaastaServiceConfigLoader
+from paasta_tools.utils import get_services_for_cluster
+from paasta_tools.utils import load_system_paasta_config
+
+
+def main() -> None:
+    system_paasta_config = load_system_paasta_config()
+
+    kube_client = KubeClient()
+
+    services = {
+        service
+        for service, instance in get_services_for_cluster(
+            cluster=system_paasta_config.get_cluster(), instance_type="kubernetes"
+        )
+    }
+
+    for service in services:
+        pscl = PaastaServiceConfigLoader(service=service, load_deployments=False)
+        for instance_config in pscl.instance_configs(
+            cluster=system_paasta_config.get_cluster(),
+            instance_type_class=KubernetesDeploymentConfig,
+        ):
+            max_instances = instance_config.get_max_instances()
+            if max_instances is not None:
+                formatted_application = instance_config.format_kubernetes_app()
+                formatted_application.spec.replicas = max_instances
+                wrapper = get_application_wrapper(formatted_application)
+                wrapper.soa_config = instance_config
+                print(f"Scaling up {service}.{instance_config.instance}")
+                wrapper.update(kube_client)
+
+
+if __name__ == "__main__":
+    main()

paasta_tools/autoscaling/pause_service_autoscaler.py

@@ -0,0 +1,77 @@
+import os
+import time
+from datetime import datetime
+
+import pytz
+from tzlocal import get_localzone as tzlocal_get_localzone
+
+import paasta_tools.paastaapi.models as paastamodels
+from paasta_tools.api import client
+
+
+def get_localzone():
+    if "TZ" in os.environ:
+        return pytz.timezone(os.environ["TZ"])
+    else:
+        return tzlocal_get_localzone()
+
+
+def print_paused_message(pause_time):
+    local_tz = get_localzone()
+    paused_readable = local_tz.localize(datetime.fromtimestamp(pause_time)).strftime(
+        "%F %H:%M:%S %Z"
+    )
+    print(f"Service autoscaler is paused until {paused_readable}")
+
+
+def get_service_autoscale_pause_time(cluster):
+    api = client.get_paasta_oapi_client(cluster=cluster, http_res=True)
+    if not api:
+        print("Could not connect to paasta api. Maybe you misspelled the cluster?")
+        return 1
+    pause_time, status, _ = api.default.get_service_autoscaler_pause(
+        _return_http_data_only=False
+    )
+    if status == 500:
+        print("Could not connect to zookeeper server")
+        return 2
+
+    pause_time = float(pause_time)
+    if pause_time < time.time():
+        print("Service autoscaler is not paused")
+    else:
+        print_paused_message(pause_time)
+
+    return 0
+
+
+def update_service_autoscale_pause_time(cluster, mins):
+    api = client.get_paasta_oapi_client(cluster=cluster, http_res=True)
+    if not api:
+        print("Could not connect to paasta api. Maybe you misspelled the cluster?")
+        return 1
+    res, status, _ = api.default.update_service_autoscaler_pause(
+        paastamodels.InlineObject(minutes=int(mins)), _return_http_data_only=False
+    )
+    if status == 500:
+        print("Could not connect to zookeeper server")
+        return 2
+
+    print(f"Service autoscaler is paused for {mins} minutes")
+    return 0
+
+
+def delete_service_autoscale_pause_time(cluster):
+    api = client.get_paasta_oapi_client(cluster=cluster, http_res=True)
+    if not api:
+        print("Could not connect to paasta api. Maybe you misspelled the cluster?")
+        return 1
+    res, status, _ = api.default.delete_service_autoscaler_pause(
+        _return_http_data_only=False
+    )
+    if status == 500:
+        print("Could not connect to zookeeper server")
+        return 2
+
+    print("Service autoscaler is unpaused")
+    return 0

paasta_tools/autoscaling/utils.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from collections import defaultdict
+from typing import Callable
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import TypedDict
+
+
+_autoscaling_components: Dict[str, Dict[str, Callable]] = defaultdict(dict)
+
+
+def register_autoscaling_component(name, method_type):
+    def outer(autoscaling_method):
+        _autoscaling_components[method_type][name] = autoscaling_method
+        return autoscaling_method
+
+    return outer
+
+
+def get_autoscaling_component(name, method_type):
+    return _autoscaling_components[method_type][name]
+
+
+class MetricsProviderDict(TypedDict, total=False):
+    type: str
+    decision_policy: str
+    setpoint: float
+    desired_active_requests_per_replica: int
+    forecast_policy: Optional[str]
+    moving_average_window_seconds: Optional[int]
+    use_resource_metrics: bool
+    prometheus_adapter_config: Optional[dict]
+    max_instances_alert_threshold: float
+
+
+class AutoscalingParamsDict(TypedDict, total=False):
+    metrics_providers: List[MetricsProviderDict]
+    scaledown_policies: Optional[dict]