PyObservability 0.0.3__py3-none-any.whl → 1.0.0__py3-none-any.whl

pyobservability/config/settings.py

@@ -2,14 +2,40 @@ import json
 import os
 import pathlib
 import socket
-from typing import List
+from typing import Any, Dict, List
 
 import yaml
-from pydantic import BaseModel, Field, HttpUrl, PositiveInt
+from pydantic import BaseModel, Field, FilePath, HttpUrl, PositiveInt
 from pydantic.aliases import AliasChoices
 from pydantic_settings import BaseSettings
 
 
+def detailed_log_config() -> Dict[str, Any]:
+    return {
+        "version": 1,
+        "disable_existing_loggers": False,
+        "formatters": {
+            "default": {
+                "format": "%(asctime)s - %(levelname)s - [%(module)s:%(lineno)d] - %(funcName)s - %(message)s",
+                "datefmt": "%b-%d-%Y %I:%M:%S %p",
+            }
+        },
+        "handlers": {
+            "default": {
+                "class": "logging.StreamHandler",
+                "formatter": "default",
+                "stream": "ext://sys.stdout",
+            }
+        },
+        "loggers": {
+            "uvicorn": {"handlers": ["default"], "level": "INFO"},
+            "uvicorn.error": {"handlers": ["default"], "level": "INFO", "propagate": False},
+            "uvicorn.access": {"handlers": ["default"], "level": "INFO", "propagate": False},
+        },
+        "root": {"handlers": ["default"], "level": "INFO"},
+    }
+
+
 class PydanticEnvConfig(BaseSettings):
     """Pydantic BaseSettings with custom order for loading environment variables.
 
@@ -63,6 +89,8 @@ class EnvConfig(PydanticEnvConfig):
     targets: List[MonitorTarget] = Field(..., validation_alias=alias_choices("TARGETS"))
     interval: PositiveInt = Field(3, validation_alias=alias_choices("INTERVAL"))
 
+    log_config: Dict[str, Any] | FilePath | None = None
+
     username: str | None = Field(None, validation_alias=alias_choices("USERNAME"))
     password: str | None = Field(None, validation_alias=alias_choices("PASSWORD"))
 
@@ -124,3 +152,4 @@ def env_loader(**kwargs) -> EnvConfig:
 
 
 env: EnvConfig
+targets_by_url: Dict[str, Dict[str, str]]

pyobservability/main.py

@@ -4,13 +4,14 @@ import warnings
 
 import uiauth
 import uvicorn
-from fastapi import FastAPI, Request, WebSocket, WebSocketDisconnect
+from fastapi import FastAPI, Request
 from fastapi.routing import APIRoute, APIWebSocketRoute
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 
 from pyobservability.config import enums, settings
-from pyobservability.monitor import Monitor
+from pyobservability.transport import websocket_endpoint
+from pyobservability.version import __version__
 
 LOGGER = logging.getLogger("uvicorn.default")
 
@@ -32,33 +33,9 @@ async def index(request: Request):
     Args:
         request: FastAPI request object.
     """
-    return templates.TemplateResponse("index.html", {"request": request, "targets": settings.env.targets})
-
-
-async def websocket_endpoint(websocket: WebSocket):
-    """Websocket endpoint to render the metrics.
-
-    Args:
-        websocket: FastAPI websocket object.
-    """
-    monitor = Monitor(targets=settings.env.targets, poll_interval=settings.env.interval)
-    await websocket.accept()
-    await monitor.start()
-    q = monitor.subscribe()
-    try:
-        while True:
-            payload = await q.get()
-            # send as JSON text
-            await websocket.send_json(payload)
-    except WebSocketDisconnect:
-        monitor.unsubscribe(q)
-    except Exception:
-        monitor.unsubscribe(q)
-        try:
-            await websocket.close()
-        except Exception as err:
-            LOGGER.warning(err)
-    await monitor.stop()
+    return templates.TemplateResponse(
+        "index.html", {"request": request, "targets": settings.env.targets, "version": __version__}
+    )
 
 
 def include_routes():
@@ -100,13 +77,16 @@ def include_routes():
 
 def start(**kwargs):
     settings.env = settings.env_loader(**kwargs)
-    settings.env.targets = [
-        {k: str(v) for k, v in target.model_dump().items()} for target in settings.env.targets
-    ]
+    settings.env.targets = [{k: str(v) for k, v in target.model_dump().items()} for target in settings.env.targets]
+    settings.targets_by_url = {t["base_url"]: t for t in settings.env.targets}
     include_routes()
     uvicorn_args = dict(
         host=settings.env.host,
         port=settings.env.port,
         app=PyObservability,
     )
+    if settings.env.log_config:
+        uvicorn_args["log_config"] = (
+            settings.env.log_config if isinstance(settings.env.log_config, dict) else str(settings.env.log_config)
+        )
     uvicorn.run(**uvicorn_args)

pyobservability/monitor.py

@@ -1,85 +1,64 @@
-# app/monitor.py
-
 import asyncio
+import json
 import logging
 from asyncio import CancelledError
+from collections.abc import Generator
 from typing import Any, Dict, List
-from urllib.parse import urlparse
 
 import aiohttp
 
+from pyobservability.config import settings
+
 LOGGER = logging.getLogger("uvicorn.default")
+OBS_PATH = "/observability"
 
-###############################################################################
-# ENDPOINT DEFINITIONS (PyNinja Correct)
-###############################################################################
-
-ENDPOINTS = {
-    "ip": {
-        "path": "/get-ip",
-        "params": {"public": "false"},
-    },
-    "cpu": {
-        "path": "/get-cpu",
-        "params": {"interval": 2, "per_cpu": "true"},
-    },
-    "cpu_load": {
-        "path": "/get-cpu-load",
-        "params": {},
-    },
-    "gpu": {
-        "path": "/get-processor",
-        "params": {},
-    },
-    "memory": {
-        "path": "/get-memory",
-        "params": {},
-    },
-    "disk": {
-        "path": "/get-disk-utilization",
-        "params": {"path": "/"},
-    },
-    "disks": {
-        "path": "/get-all-disks",
-        "params": {},
-    },
-    "services": {
-        "path": "/get-all-services",
-        "params": {},
-    },
-    "docker_stats": {
-        "path": "/get-docker-stats",
-        "params": {},
-    },
-    "certificates": {
-        "path": "/get-certificates",
-        "params": {},
-    },
-}
-
-
-###############################################################################
-# MONITOR CLASS
-###############################################################################
 
+def refine_service(service_list: List[Dict[str, Any]]) -> Generator[Dict[str, Dict[str, str]]]:
+    for service in service_list:
+        service["memory"] = dict(rss=service.get("memory", {}).get("rss"), vms=service.get("memory", {}).get("vms"))
+        service["cpu"] = dict(
+            user=round(service.get("cpu", {}).get("user", 0), 2),
+            system=round(service.get("cpu", {}).get("system", 0), 2),
+        )
+        yield service
 
-class Monitor:
 
-    def __init__(self, targets: List[Dict[str, str]], poll_interval: float):
-        self.targets = targets
-        self.poll_interval = poll_interval
-        self.sessions: Dict[str, aiohttp.ClientSession] = {}
-        self._ws_subscribers: List[asyncio.Queue] = []
-        self._task = None
+class Monitor:
+    def __init__(self, target: Dict[str, str]):
+        self.name = target["name"]
+        self.base_url = target["base_url"]
+        self.apikey = target["apikey"]
+        self.flags = {"all_services": False}
+
+        self.session: aiohttp.ClientSession | None = None
+        self._task: asyncio.Task | None = None
         self._stop = asyncio.Event()
 
-    ############################################################################
-    # LIFECYCLE
-    ############################################################################
+        self._ws_subscribers = []  # list of asyncio.Queue
+
+    # ------------------------------
+    # SUBSCRIBE / UNSUBSCRIBE
+    # ------------------------------
+    def subscribe(self) -> asyncio.Queue:
+        q = asyncio.Queue(maxsize=10)
+        self._ws_subscribers.append(q)
+        return q
+
+    def unsubscribe(self, q: asyncio.Queue):
+        if q in self._ws_subscribers:
+            self._ws_subscribers.remove(q)
+
+    # ------------------------------
+    # START / STOP
+    # ------------------------------
     async def start(self):
-        for target in self.targets:
-            self.sessions[target["base_url"]] = aiohttp.ClientSession()
-        self._task = asyncio.create_task(self._run_loop())
+        if self._task:
+            return  # already running
+
+        self._stop.clear()
+        self.session = aiohttp.ClientSession()
+
+        self._task = asyncio.create_task(self._stream_target())
 
     async def stop(self):
         self._stop.set()
@@ -89,110 +68,109 @@ class Monitor:
                 await self._task
             except CancelledError:
                 pass
-
-        for sess in self.sessions.values():
-            await sess.close()
-
-    ############################################################################
-    # SUBSCRIBE / UNSUBSCRIBE
-    ############################################################################
-    def subscribe(self) -> asyncio.Queue:
-        q = asyncio.Queue(maxsize=10)
-        self._ws_subscribers.append(q)
-        return q
-
-    def unsubscribe(self, q: asyncio.Queue):
-        try:
-            self._ws_subscribers.remove(q)
-        except ValueError:
-            pass
-
-    ############################################################################
-    # FETCH WRAPPER
-    ############################################################################
-    async def _fetch(self, session, base_url, ep, headers: Dict[str, str], params=None):
-        url = base_url.rstrip("/") + ep
-        try:
-            async with session.get(url, headers=headers, params=params, timeout=10) as resp:
-                if resp.status == 200:
-                    try:
-                        return await resp.json()
-                    except Exception as err:
-                        LOGGER.debug(err)
-                        return
-                parsed = urlparse(url)
-                LOGGER.debug("Exception on '%s' - [%d]: %s", parsed.path, resp.status, await resp.text())
+            self._task = None
+
+        if self.session:
+            await self.session.close()
+            self.session = None
+
+    async def update_flags(self, **kwargs):
+        for k, v in kwargs.items():
+            if k in self.flags:
+                self.flags[k] = v
+
+        # restart stream with new params
+        await self._restart_stream()
+
+    async def _restart_stream(self):
+        await self.stop()
+        await self.start()
+
+    # ------------------------------
+    # FETCH STREAM
+    # ------------------------------
+    async def _fetch_stream(self):
+        query = f"?interval={settings.env.interval}"
+        if self.flags["all_services"]:
+            query += "&all_services=true"
+        url = self.base_url.rstrip("/") + OBS_PATH + query
+        headers = {"Accept": "application/json", "Authorization": f"Bearer {self.apikey}"}
+
+        async with self.session.get(
+            url, headers=headers, timeout=aiohttp.ClientTimeout(total=None, connect=3, sock_read=None, sock_connect=3)
+        ) as resp:
+            if resp.status != 200:
+                LOGGER.error("Bad response [%d] from %s", resp.status, url)
                 return
-        except Exception as err:
-            LOGGER.debug(err)
-            return
-
-    ############################################################################
-    # PER-TARGET POLLING
-    ############################################################################
-    async def _poll_target(self, target: Dict[str, Any]) -> Dict[str, Any]:
-        base = target["base_url"]
-        apikey = target["apikey"]
-        session = self.sessions[base]
-        headers = {"Accept": "application/json", "Authorization": f"Bearer {apikey}"}
-
-        result = {"name": target["name"], "base_url": base, "metrics": {}}
-
-        # Fire ALL requests concurrently
-        tasks = {}
-
-        for key, cfg in ENDPOINTS.items():
-            tasks[key] = asyncio.create_task(
-                self._fetch(session, base, cfg["path"], headers=headers, params=cfg["params"])
-            )
-
-        # Wait for all endpoints
-        raw_results = await asyncio.gather(*tasks.values(), return_exceptions=True)
-
-        for (key, _), resp in zip(tasks.items(), raw_results):
-            if isinstance(resp, Exception):
-                result["metrics"][key] = None
-                continue
-            if isinstance(resp, dict):
-                result["metrics"][key] = resp.get("detail", resp)
-            else:
-                # raw string / number / list / etc
-                result["metrics"][key] = resp
-
-        return result
-
-    ############################################################################
-    # POLL ALL HOSTS
-    ############################################################################
-    async def _poll_all(self) -> List[Dict[str, Any]]:
-        tasks = [self._poll_target(target) for target in self.targets]
-        results = await asyncio.gather(*tasks, return_exceptions=True)
-        out = []
-        for r in results:
-            if isinstance(r, Exception):
-                LOGGER.error("%s", r)
-                out.append({"error": str(r)})
-            else:
-                out.append(r)
-        return out
-
-    ############################################################################
-    # MAIN LOOP
-    ############################################################################
-    async def _run_loop(self):
-        while not self._stop.is_set():
-            metrics = await self._poll_all()
 
-            payload = {"type": "metrics", "ts": asyncio.get_event_loop().time(), "data": metrics}
+            async for raw in resp.content:
+                line = raw.decode().strip()
+                if not line:
+                    continue
 
-            for q in list(self._ws_subscribers):
                 try:
-                    q.put_nowait(payload)
-                except asyncio.QueueFull:
+                    parsed = json.loads(line)
                     try:
-                        _ = q.get_nowait()
-                        q.put_nowait(payload)
-                    except Exception:
-                        pass
+                        if service_stats := parsed.get("service_stats"):
+                            parsed["service_stats"] = list(refine_service(service_stats))
+                    except Exception as error:
+                        LOGGER.error("Received [%s: %s] when parsing services for %s", type(error), error, self.name)
+                    yield parsed
+                except json.JSONDecodeError:
+                    LOGGER.debug("Bad JSON from %s: %s", self.base_url, line)
+
+    # ------------------------------
+    # STREAM LOOP
+    # ------------------------------
+    async def _stream_target(self):
+        errors = {}
+        while not self._stop.is_set():
+            try:
+                async for payload in self._fetch_stream():
+                    result = {
+                        "type": "metrics",
+                        "ts": asyncio.get_event_loop().time(),
+                        "data": [
+                            {
+                                "name": self.base_url,
+                                "base_url": self.base_url,
+                                "metrics": payload,
+                            }
+                        ],
+                    }
+
+                    for q in list(self._ws_subscribers):
+                        try:
+                            q.put_nowait(result)
+                        except asyncio.QueueFull:
+                            _ = q.get_nowait()
+                            q.put_nowait(result)
+            except Exception as err:
+                LOGGER.debug("Stream error for %s: %s", self.base_url, err)
+                if errors.get(self.base_url):
+                    if errors[self.base_url] < 10:
+                        errors[self.base_url] += 1
+                    else:
+                        LOGGER.error(err.with_traceback())
+                        LOGGER.error("%s exceeded error threshold.", self.base_url)
+
+                        # notify subscribers before stopping
+                        error_msg = {
+                            "type": "error",
+                            "base_url": self.base_url,
+                            "message": f"{self.name!r} is unreachable.",
+                        }
+
+                        for q in list(self._ws_subscribers):
+                            try:
+                                q.put_nowait(error_msg)
+                            except asyncio.QueueFull as warn:
+                                LOGGER.warning(warn)
+                                _ = q.get_nowait()
+                                q.put_nowait(result)
+                        await self.stop()
+                        return
+                else:
+                    errors[self.base_url] = 1
 
-            await asyncio.sleep(self.poll_interval)
+            await asyncio.sleep(1)