paasta-tools 1.21.3__py3-none-any.whl

paasta_tools/metrics/metrics_lib.py

@@ -0,0 +1,217 @@
+import logging
+import os
+import socket
+import time
+from abc import ABC
+from abc import abstractmethod
+from inspect import currentframe
+from types import TracebackType
+from typing import Any
+from typing import Callable
+from typing import Dict
+from typing import Optional
+from typing import Type
+from typing import Union
+
+from typing_extensions import Protocol
+
+from paasta_tools.utils import load_system_paasta_config
+
+log = logging.getLogger(__name__)
+
+try:
+    import yelp_meteorite
+except ImportError:
+    yelp_meteorite = None
+
+_metrics_interfaces: Dict[str, Type["BaseMetrics"]] = {}
+
+
+class TimerProtocol(Protocol):
+    def __enter__(self) -> "TimerProtocol":
+        raise NotImplementedError()
+
+    def __exit__(
+        self,
+        err_type: Optional[type],
+        value: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> None:
+        raise NotImplementedError()
+
+    def __call__(
+        self,
+    ) -> Optional[float]:
+        raise NotImplementedError()
+
+    def start(self) -> None:
+        raise NotImplementedError()
+
+    def stop(self, **kwargs: Any) -> None:
+        raise NotImplementedError()
+
+    def record(self, value: float, **kwargs: Any) -> None:
+        raise NotImplementedError()
+
+
+class GaugeProtocol(Protocol):
+    def set(self, value: Union[int, float]) -> None:
+        raise NotImplementedError()
+
+
+class CounterProtocol(Protocol):
+    def count(self) -> None:
+        raise NotImplementedError()
+
+
+class BaseMetrics(ABC):
+    def __init__(self, base_name: str) -> None:
+        self.base_name = base_name
+
+    @abstractmethod
+    def create_timer(self, name: str, **kwargs: Any) -> TimerProtocol:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def create_gauge(self, name: str, **kwargs: Any) -> GaugeProtocol:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def create_counter(self, name: str, **kwargs: Any) -> CounterProtocol:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def emit_event(self, name: str, **kwargs: Any) -> bool:
+        raise NotImplementedError()
+
+
+def get_metrics_interface(base_name: str) -> BaseMetrics:
+    metrics_provider = load_system_paasta_config().get_metrics_provider()
+    return _metrics_interfaces[metrics_provider](base_name)
+
+
+def register_metrics_interface(
+    name: Optional[str],
+) -> Callable[[Type[BaseMetrics]], Type[BaseMetrics]]:
+    def outer(func: Type[BaseMetrics]) -> Type[BaseMetrics]:
+        _metrics_interfaces[name] = func
+        return func
+
+    return outer
+
+
+@register_metrics_interface("meteorite")
+class MeteoriteMetrics(BaseMetrics):
+    def __init__(self, base_name: str) -> None:
+        self.base_name = base_name
+        if yelp_meteorite is None:
+            raise ImportError(
+                "yelp_meteorite not imported, please try another metrics provider"
+            )
+
+    def create_timer(self, name: str, **kwargs: Any) -> TimerProtocol:
+        return yelp_meteorite.create_timer(self.base_name + "." + name, **kwargs)
+
+    def create_gauge(self, name: str, **kwargs: Any) -> GaugeProtocol:
+        return yelp_meteorite.create_gauge(self.base_name + "." + name, **kwargs)
+
+    def create_counter(self, name: str, **kwargs: Any) -> CounterProtocol:
+        return yelp_meteorite.create_counter(self.base_name + "." + name, **kwargs)
+
+    def emit_event(self, name: str, **kwargs: Any) -> bool:
+        return yelp_meteorite.emit_event(self.base_name + "." + name, **kwargs)
+
+
+class Timer(TimerProtocol):
+    def __init__(self, name: str) -> None:
+        self.name = name
+        self.elapsed_ms: Optional[float] = None
+
+    def __enter__(self) -> TimerProtocol:
+        self.start()
+        return self
+
+    def __exit__(
+        self,
+        err_type: Optional[type],
+        value: Optional[BaseException],
+        traceback: Optional[TracebackType],
+    ) -> None:
+        if not err_type:
+            self.stop()
+
+    def __call__(self) -> Optional[float]:
+        if self.elapsed_ms is None:
+            self.stop()
+        return self.elapsed_ms
+
+    def start(self) -> None:
+        log.debug("timer {} start at {}".format(self.name, time.time()))
+
+    def stop(self, **kwargs: Any) -> None:
+        log.debug("timer {} stop at {}".format(self.name, time.time()))
+
+    def record(self, value: float, **kwargs: Any) -> None:
+        log.debug(f"timer {self.name} record value {value}")
+
+
+class Gauge(GaugeProtocol):
+    def __init__(self, name: str) -> None:
+        self.name = name
+
+    def set(self, value: Union[int, float]) -> None:
+        log.debug(f"gauge {self.name} set to {value}")
+
+
+class Counter(GaugeProtocol):
+    def __init__(self, name: str) -> None:
+        self.name = name
+        self.counter = 0
+
+    def count(self) -> None:
+        self.counter += 1
+        log.debug(f"counter {self.name} incremented to {self.counter}")
+
+
+@register_metrics_interface(None)
+class NoMetrics(BaseMetrics):
+    def __init__(self, base_name: str) -> None:
+        self.base_name = base_name
+
+    def create_timer(self, name: str, **kwargs: Any) -> Timer:
+        return Timer(self.base_name + "." + name)
+
+    def create_gauge(self, name: str, **kwargs: Any) -> Gauge:
+        return Gauge(self.base_name + "." + name)
+
+    def create_counter(self, name: str, **kwargs: Any) -> Counter:
+        return Counter(self.base_name + "." + name)
+
+    def emit_event(self, name: str, **kwargs: Any) -> bool:
+        log.debug(f"event {name} occurred with properties: {kwargs}")
+        return True
+
+
+def system_timer(
+    name: str = "system_process_duration", dimensions: Dict[str, Any] = {}
+) -> TimerProtocol:
+    metrics_interface = get_metrics_interface("paasta")
+    parent_pid = os.getppid()
+    pid = os.getpid()
+    hostname = socket.gethostname()
+    # set our parent caller as the path dimension, e.g. /path/to/setup_kubernetes_job
+    current = currentframe()
+    parent = current.f_back
+    path = parent.f_globals.get("__file__", "unknown")
+    # Note any of these dimensions may be overridden by caller
+    default_dimensions = {
+        "path": path,
+        "host": hostname,
+        # ppid is included given some system processes are called multiple times in parallel for a single 'run'
+        "parent_pid": parent_pid,
+        "pid": pid,
+    }
+    default_dimensions.update(dimensions)
+    return metrics_interface.create_timer(
+        name=name, default_dimensions=default_dimensions
+    )

paasta_tools/monitoring/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

paasta_tools/monitoring/check_k8s_api_performance.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+# Copyright 2015-2016 Yelp Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import time
+
+from kubernetes import client
+
+from paasta_tools.kubernetes_tools import KUBE_CONFIG_USER_PATH
+from paasta_tools.kubernetes_tools import KubeClient
+
+
+NAMESPACE = "paasta"
+LABEL_SELECTOR = None
+
+
+def parse_k8s_api_performance_options() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+
+    parser.add_argument(
+        "--kube-config-path",
+        "-k",
+        dest="kube_config_path",
+        default=KUBE_CONFIG_USER_PATH,
+        help="Specify the path to the kubeconfig file",
+    )
+
+    parser.add_argument(
+        "--context",
+        "-c",
+        dest="context",
+        default=None,
+        help="Pass the cluster you want to connect to",
+    )
+
+    parser.add_argument(
+        "--namespace",
+        "-n",
+        dest="namespace",
+        default=NAMESPACE,
+        help="Pass the namespace you want to query",
+    )
+
+    parser.add_argument(
+        "--label-selector",
+        "-l",
+        dest="label_selector",
+        default=LABEL_SELECTOR,
+        help="Pass the label selector you want to select",
+    )
+
+    options = parser.parse_args()
+    return options
+
+
+def get_pods_from_namespace(
+    kube_client: KubeClient, namespace: str
+) -> client.models.v1_pod_list.V1PodList:
+
+    # list all the running pods
+    ret = kube_client.core.list_namespaced_pod(namespace, watch=False)
+    return ret
+
+
+def get_pods_from_namespace_selector(
+    kube_client: KubeClient, namespace: str, label_selector: str
+) -> client.models.v1_pod_list.V1PodList:
+
+    # list all the running pods
+    ret = kube_client.core.list_namespaced_pod(
+        namespace, watch=False, label_selector=label_selector
+    )
+    return ret
+
+
+if __name__ == "__main__":
+
+    options = parse_k8s_api_performance_options()
+    kube_client = KubeClient(
+        config_file=options.kube_config_path, context=options.context
+    )
+
+    # measure the performance of this function
+    start = time.time()
+    ret = get_pods_from_namespace_selector(
+        kube_client, options.namespace, options.label_selector
+    )
+    end = time.time()
+
+    print("time it took to run this function: ", end - start)
+
+    # pods are stored here
+    pods = ret.items
+    pods_names = [pod.metadata.name for pod in pods]
+    pods_status = [pod.status.phase for pod in pods]
+
+    print(list(zip(pods_names, pods_status)))