polaris-python 0.0.1.dev3__py3-none-any.whl

polaris/plugin/stat_reporter/prometheus.py

@@ -0,0 +1,375 @@
+# Tencent is pleased to support the open source community by making polaris-python available.
+#
+# Copyright (C) 2021 Tencent. All rights reserved.
+#
+# Licensed under the BSD 3-Clause License (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://opensource.org/licenses/BSD-3-Clause
+#
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+
+"""Java-compatible period gauges and Prometheus pull / Pushgateway export."""
+
+import asyncio
+import gzip
+import random
+import re
+import time
+from threading import RLock
+from urllib.parse import quote, urlsplit, urlunsplit
+
+from aiohttp import ClientSession, ClientTimeout, web
+from prometheus_client import CollectorRegistry, Gauge, generate_latest
+from prometheus_client.exposition import CONTENT_TYPE_LATEST
+
+from polaris.pkg.model.error import ErrorCode, SDKError
+from polaris.pkg.config import stat_reporter as stat_reporter_config
+from polaris.pkg.model.service import GetOneInstanceRequest, RetStatus
+from polaris.plugin_manager import PluginType, register_plugin
+from .base import BaseStatReporter
+
+LABELS = ("callee_namespace", "callee_service", "callee_method", "callee_instance",
+          "callee_instance_type", "callee_result_code", "callee_result", "caller_labels",
+          "caller_namespace", "caller_service", "caller_ip", "metric_name")
+METRICS = {
+    "upstream_rq_total": "total of request per period",
+    "upstream_rq_success": "total of success request per period",
+    "upstream_rq_timeout": "total of request delay per period",
+    "upstream_rq_max_timeout": "maximum request delay per period",
+    "upstream_rq_min_timeout": "minimum request delay per period",
+}
+PULL_INTERVAL_SECONDS = 30
+REVISION_LOG_INTERVAL_SECONDS = 30 * 60
+PUSH_ERROR_LOG_INTERVAL_SECONDS = 30
+NULL = "__NULL__"
+
+
+@register_plugin(PluginType.STAT_REPORTER, "prometheus")
+class PrometheusReporter(BaseStatReporter):
+    """Own independent metrics, HTTP listener and push task for one context."""
+
+    def __init__(self, init_ctx):
+        self.global_config = init_ctx.config.get_global()
+        self.config = self.global_config.get_stat_reporter().get_prometheus()
+        self.enabled = self.global_config.get_stat_reporter().is_enable()
+        self.client_id = init_ctx.value_ctx.get_client_id()
+        from polaris.api.log import get_logger
+        self.logger = init_ctx.logger.get_logger("statReport") if init_ctx.logger else get_logger("statReport")
+        self.registry = CollectorRegistry()
+        self.gauges = {name: Gauge(name, description, LABELS, registry=self.registry)
+                       for name, description in METRICS.items()}
+        self.lock = RLock()
+        self.period = 0
+        self.values = {}
+        self.task = None
+        self.runner = None
+        self._admin_server = None
+        self.session = None
+        self.push_count = 0
+        self._port = None
+        self._reported_port = None
+        self.engine = None
+        self.closed = False
+        self.started = False
+        self.random = random.Random()
+        self.push_lock = asyncio.Lock()
+        self.pushing = False
+        self._last_pull_revision_log = None
+        self._last_push_revision_log = None
+        self._last_push_error = ""
+        self._last_push_error_log = 0.0
+
+    @property
+    def port(self):
+        """Return the shared admin port when mounted, otherwise the private one."""
+        return self._admin_server.port if self._admin_server is not None else self._port
+
+    def report(self, result):
+        """Record Java label values and period count / delay aggregations."""
+        if not self.enabled or self.closed:
+            return
+        method = result.method
+        if method:
+            for pattern in self.config.path_regex_list:
+                if re.fullmatch(pattern, method):
+                    method = pattern
+                    break
+        caller = result.caller_service
+        ip = result.caller_ip
+        if ip is None:
+            ip = self.global_config.get_api().get_bind_ip()
+        labels = (result.get_namespace(), result.get_service(), method,
+                  f"{result.get_host()}:{result.get_port()}", result.instance_type,
+                  result.ret_code, RetStatus.descriptions[result.ret_status], result.labels,
+                  caller.get_namespace() if caller else None,
+                  caller.get_service() if caller else None, ip)
+        key = tuple(NULL if item is None else str(item) for item in labels)
+        delay = result.delay or 0
+        success = int(result.ret_status == RetStatus.RetSuccess)
+        with self.lock:
+            period = self.period + int(self.pushing)
+            existing = self.values.get(key)
+            if existing is None or existing[0] != period:
+                self.values[key] = (period, [1, success, delay, delay, delay])
+            else:
+                values = existing[1]
+                values[0] += 1
+                values[1] += success
+                if result.delay is not None:
+                    values[2] += delay
+                    values[3] = max(values[3], delay)
+                    values[4] = min(values[4], delay)
+
+    def aggregate(self, advance=True):
+        """Publish this period, zero stale series, remove after two idle revisions."""
+        with self.lock:
+            active_series = sum(
+                1 for revision, values in self.values.values()
+                if revision == self.period and values[0] > 0
+            )
+            request_count = sum(
+                values[0] for revision, values in self.values.values() if revision == self.period
+            )
+            for key, (revision, values) in list(self.values.items()):
+                for name, value in zip(METRICS, values):
+                    labels = (*key, NULL)
+                    if revision < self.period - 2:
+                        self.gauges[name].remove(*labels)
+                    else:
+                        self.gauges[name].labels(*labels).set(value if revision == self.period else 0)
+                if revision < self.period - 2:
+                    del self.values[key]
+            if advance:
+                self.period += 1
+            if not advance:
+                return
+            self.logger.debug(
+                "[metrics][%s] aggregate completed period=%d activeSeries=%d requests=%d retainedSeries=%d",
+                self.config.type, self.period, active_series, request_count, len(self.values),
+            )
+            now = time.monotonic()
+            if (self._last_pull_revision_log is None
+                    or now - self._last_pull_revision_log >= REVISION_LOG_INTERVAL_SECONDS):
+                self.logger.info(
+                    "[metrics][pull] revision collector inc current revision to %d", self.period,
+                )
+                self._last_pull_revision_log = now
+
+    def render(self):
+        """Return the current exported Prometheus text without advancing the period."""
+        with self.lock:
+            data = generate_latest(self.registry)
+            self.logger.debug(
+                "[metrics][%s] render completed period=%d series=%d bytes=%d",
+                self.config.type, self.period, len(self.values), len(data),
+            )
+            return data
+
+    def meta_info(self):
+        """Advertise the pull endpoint in the SDK client report, never in push mode."""
+        if not self.enabled or self.closed or self.port is None or self.config.type != "pull":
+            return None
+        port = self.port
+        if port != self._reported_port:
+            self._reported_port = port
+            self.logger.info(
+                "[metrics][pull] endpoint ready port=%s path=%s", port, self.config.path,
+            )
+        return {"target": "prometheus", "port": port, "path": self.config.path, "protocol": "http"}
+
+    async def start(self, engine):
+        """Start pull or push resources after the SDK address is resolved."""
+        self.engine = engine
+        if not self.enabled:
+            self.logger.info("[metrics] reporter disabled by configuration")
+            return
+        admin = self.global_config.get_admin()
+        if self.config.type == "pull":
+            if admin.port < 0:
+                self.logger.warning(
+                    "[metrics][pull] reporter disabled because global.admin.port < 0")
+                self.enabled = False
+                return
+            shared = getattr(engine, "admin_server", None) if engine is not None else None
+            if shared is not None:
+                shared.add_route("GET", self.config.path, self._metrics)
+                if self.config.path != "/-/healthy":
+                    shared.add_route("GET", "/-/healthy", self._metrics)
+                self._admin_server = shared
+            else:
+                app = web.Application()
+                app.router.add_get(self.config.path, self._metrics)
+                if self.config.path != "/-/healthy":
+                    app.router.add_get("/-/healthy", self._metrics)
+                self.runner = web.AppRunner(app, access_log=None)
+                await self.runner.setup()
+                listen_port = (admin.port if admin.port is not None
+                               else stat_reporter_config.DEFAULT_ADMIN_PORT)
+                site = web.TCPSite(self.runner, admin.host, listen_port)
+                try:
+                    await site.start()
+                except OSError as error:
+                    await self.runner.cleanup()
+                    self.runner = None
+                    raise SDKError(ErrorCode.ErrCodeAPIInvalidConfig,
+                                   f"Cannot bind metrics listener {admin.host}:{admin.port}: {error}") from error
+                self._port = site._server.sockets[0].getsockname()[1]
+        else:
+            self.session = ClientSession(timeout=ClientTimeout(total=self.global_config.get_api().timeout / 1000))
+        self.task = asyncio.create_task(self._run(), name="polaris-prometheus")
+        self.started = True
+        interval = self.config.push_interval / 1000 if self.config.type == "push" else PULL_INTERVAL_SECONDS
+        port = self.port if self.port is not None else admin.port
+        self.logger.info(
+            "[metrics][%s] reporter started path=%s interval=%.3fs admin=%s:%s",
+            self.config.type, self.config.path, interval, admin.host, port,
+        )
+
+    async def _metrics(self, request):
+        data = self.render()
+        self.logger.debug(
+            "[metrics][pull] served path=%s port=%s series=%d bytes=%d",
+            request.path, self.port, len(self.values), len(data),
+        )
+        return web.Response(body=data, headers={"Content-Type": CONTENT_TYPE_LATEST})
+
+    async def _address(self):
+        if self.config.address:
+            return self.random.choice(self.config.address)
+        if self.config.namespace and self.config.service:
+            namespace = self.config.namespace
+            service = self.config.service
+        else:
+            monitor = self.global_config.get_system().get_monitor_cluster()
+            if not monitor.get_namespace() or not monitor.get_service():
+                raise SDKError(
+                    ErrorCode.ErrCodeAPIInvalidConfig,
+                    "Prometheus push requires address or namespace/service from "
+                    "plugin.prometheus or global.system.monitorCluster",
+                )
+            namespace = monitor.get_namespace()
+            service = monitor.get_service()
+        node = await self.engine._async_get_one_instance(GetOneInstanceRequest(
+            namespace=namespace, service=service))
+        if node is None:
+            raise SDKError(ErrorCode.ErrCodeAPIInstanceNotFound, "No Pushgateway instance available")
+        host = node.get_host()
+        return f"[{host}]:{node.get_port()}" if ":" in host else f"{host}:{node.get_port()}"
+
+    async def push(self):
+        """POST gauges using Java job / grouping keys; failures retain the current period."""
+        async with self.push_lock:
+            await self._push()
+
+    async def _push(self):
+        address = await self._address()
+        url = urlsplit(address if "://" in address else f"http://{address}")
+        if self.config.override_host:
+            host = self.config.override_host
+            netloc = f"[{host}]" if ":" in host else host
+            if url.port:
+                netloc += f":{url.port}"
+            url = url._replace(netloc=netloc)
+        target = urlunsplit(url._replace(path=url.path.rstrip("/") +
+                           f"/metrics/job/polaris-client/instance/{quote(self.client_id, safe='')}", query=""))
+        with self.lock:
+            self.aggregate(advance=False)
+            data = self.render()
+            snapshot = {key: (revision, list(values)) for key, (revision, values) in self.values.items()}
+            self.pushing = True
+        headers = {"Content-Type": CONTENT_TYPE_LATEST}
+        if self.config.open_gzip:
+            data = gzip.compress(data)
+            headers["Content-Encoding"] = "gzip"
+        try:
+            async with self.session.post(target, data=data, headers=headers) as response:
+                await response.read()
+                if not 200 <= response.status < 300:
+                    raise SDKError(ErrorCode.ErrCodeNetworkError, f"Pushgateway returned HTTP {response.status}")
+                self.push_count += 1
+        except BaseException:
+            with self.lock:
+                for key, (revision, values) in list(self.values.items()):
+                    if revision != self.period + 1:
+                        continue
+                    previous = snapshot.get(key)
+                    if previous is not None and previous[0] == self.period:
+                        old = previous[1]
+                        values = [old[0] + values[0], old[1] + values[1], old[2] + values[2],
+                                  max(old[3], values[3]), min(old[4], values[4])]
+                    self.values[key] = (self.period, values)
+            raise
+        else:
+            with self.lock:
+                self.period += 1
+            if self._last_push_error:
+                self.logger.info(
+                    "[metrics][push] push metrics to pushgateway recovered (was failing: %s)",
+                    self._last_push_error,
+                )
+                self._last_push_error = ""
+            self.logger.debug("[metrics][push] exported period=%d series=%d target=%s",
+                              self.period, len(snapshot), target)
+            now = time.monotonic()
+            if (self._last_push_revision_log is None
+                    or now - self._last_push_revision_log >= REVISION_LOG_INTERVAL_SECONDS):
+                self.logger.info(
+                    "[metrics][push] revision collector inc current revision to %d", self.period,
+                )
+                self._last_push_revision_log = now
+        finally:
+            with self.lock:
+                self.pushing = False
+
+    async def _run(self):
+        interval = self.config.push_interval / 1000 if self.config.type == "push" else PULL_INTERVAL_SECONDS
+        while True:
+            await asyncio.sleep(interval)
+            try:
+                if self.config.type == "push":
+                    await self.push()
+                else:
+                    self.aggregate()
+            except Exception as error:
+                if self.config.type == "push":
+                    self._log_push_error(error)
+                else:
+                    self.logger.warning("Metrics export failed; retaining current period: %s", error)
+
+    def _log_push_error(self, error):
+        error_text = str(error)
+        now = time.monotonic()
+        if (error_text != self._last_push_error
+                or now - self._last_push_error_log >= PUSH_ERROR_LOG_INTERVAL_SECONDS):
+            if not self._last_push_error:
+                self.logger.error("push metrics to pushgateway fail: %s", error_text)
+            else:
+                self.logger.warning(
+                    "push metrics to pushgateway still failing (last error unchanged for %.0fs): %s",
+                    now - self._last_push_error_log, error_text,
+                )
+            self._last_push_error_log = now
+        self._last_push_error = error_text
+
+    async def close(self):
+        """Cancel the task before closing its listener and outgoing HTTP session."""
+        self.closed = True
+        self._reported_port = None
+        if self.task is not None:
+            self.task.cancel()
+            await asyncio.gather(self.task, return_exceptions=True)
+            self.task = None
+        if self.runner is not None:
+            await self.runner.cleanup()
+            self.runner = None
+        if self.session is not None:
+            await self.session.close()
+            self.session = None
+        if self.started:
+            self.started = False
+            self.logger.info("[metrics][%s] reporter stopped", self.config.type)

polaris/plugin_manager.py

@@ -0,0 +1,146 @@
+# Tencent is pleased to support the open source community by making polaris-python available.
+#
+# Copyright (C) 2021 Tencent. All rights reserved.
+#
+# Licensed under the BSD 3-Clause License (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://opensource.org/licenses/BSD-3-Clause
+#
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+
+from threading import RLock
+
+from polaris.pkg.model.context import ValueContext
+from polaris.pkg.config.api import Configuration
+from polaris.pkg.model.error import ErrorCode, SDKError
+
+
+class PluginType:
+    SERVER_CONNECTOR = 2
+    SERVICE_ROUTER = 3
+    LOAD_BALANCE = 4
+    CIRCUIT_BREAKER = 5
+    LOCAL_CACHE = 6
+    STAT_REPORTER = 7
+    SKILL_CONNECTOR = 8
+    EVENT_REPORTER = 9
+
+
+class EventType:
+    ON_LOCALVALUE_CREATED = 1
+
+
+class PluginManager:
+    """Own plugin instances and event subscribers for one SDK context."""
+
+    def __init__(self):
+        self.plugin_container = {}
+        self.plugin_instance = {}
+        self.event_handle_map = {}
+        self._lock = RLock()
+
+    def register(self, plugin_type: PluginType, plugin_name: str, plugin_cls):
+        """Register a plugin constructor without creating a runtime instance."""
+        with self._lock:
+            self.plugin_container.setdefault(plugin_type, {})[plugin_name] = plugin_cls
+
+    def new_context_manager(self):
+        """Copy registered constructors into an empty, independent runtime manager."""
+        manager = PluginManager()
+        with self._lock:
+            manager.plugin_container = {typ: dict(plugins) for typ, plugins in self.plugin_container.items()}
+        return manager
+
+    def get(self, plugin_type: PluginType, plugin_name: str):
+        """Return a registered plugin constructor."""
+        if plugin_type in self.plugin_container:
+            return self.plugin_container[plugin_type][plugin_name]
+        else:
+            return ""
+
+    def find_instance(self, plugin_type: PluginType, plugin_name: str) -> bool:
+        """Check whether this context has created a plugin instance."""
+        if plugin_type in self.plugin_instance and plugin_name in self.plugin_instance[plugin_type]:
+            return True
+        return False
+
+    def get_instance(self, plugin_type: PluginType, plugin_name: str):
+        """Return this context's plugin instance or a configuration diagnostic."""
+        try:
+            return self.plugin_instance[plugin_type][plugin_name]
+        except KeyError as error:
+            raise SDKError(ErrorCode.ErrCodePluginError,
+                           f"Plugin is not available: {plugin_type}:{plugin_name}") from error
+
+    def set_instance(self, plugin_type: PluginType, plugin_name: str, instance):
+        """Store an instance owned exclusively by this context."""
+        if plugin_type not in self.plugin_instance:
+            self.plugin_instance[plugin_type] = {}
+
+        self.plugin_instance[plugin_type][plugin_name] = instance
+
+    def clear_instances(self):
+        """Forget this context's instances and their event subscribers."""
+        self.plugin_instance.clear()
+        self.event_handle_map.clear()
+
+    def destroy_plugins(self):
+        """Release synchronous plugin resources after the owner loop has stopped."""
+        instances = [instance for plugins in self.plugin_instance.values() for instance in plugins.values()]
+        self.clear_instances()
+        first_error = None
+        for instance in reversed(instances):
+            destroy = getattr(instance, "destroy", None)
+            if destroy is not None:
+                try:
+                    destroy()
+                except Exception as error:
+                    if first_error is None:
+                        first_error = error
+        if first_error is not None:
+            raise first_error
+
+    def register_event_handle(self, event_type, handler):
+        """Subscribe a context-owned handler to a plugin event."""
+        self.event_handle_map.setdefault(event_type, []).append(handler)
+
+    def get_event_subscriber(self, event_type):
+        """Return only this context's subscribers, or an empty tuple."""
+        return tuple(self.event_handle_map.get(event_type, ()))
+
+
+def register_plugin(plugin_type: PluginType, plugin_name: str):
+    def decorator(cls):
+        DefaultPluginManager.register(plugin_type, plugin_name, cls)
+        return cls
+
+    return decorator
+
+
+# Process-wide registration stores constructors only, never SDK runtime state.
+DefaultPluginManager = PluginManager()
+
+
+class InitContext:
+    """Provide configuration and context-owned dependencies during plugin creation."""
+
+    def __init__(self, config: Configuration = None, value_ctx: ValueContext = None,
+                 plugins: PluginManager = None):
+        self.config = config
+        self.value_ctx = value_ctx
+        self.plugins = plugins if plugins is not None else PluginManager()
+        self.logger = getattr(value_ctx, "logger", None)
+
+
+def setup(init_ctx: InitContext):
+    """Create plugins in the supplied context manager, leaving the registry untouched."""
+    manager = init_ctx.plugins
+    for plugin_type, plugins in manager.plugin_container.items():
+        for plugin_name, plugin in plugins.items():
+            if not manager.find_instance(plugin_type, plugin_name):
+                manager.set_instance(plugin_type, plugin_name, plugin(init_ctx))