cledar-sdk 2.0.2__py3-none-any.whl → 2.0.3__py3-none-any.whl

cledar/logging/tests/test_universal_plaintext_formatter.py

@@ -0,0 +1,249 @@
+# pylint: disable=unused-argument, protected-access
+import logging
+import os
+import tempfile
+from pathlib import Path
+
+import pytest
+
+from cledar.logging.universal_plaintext_formatter import UniversalPlaintextFormatter
+
+
+@pytest.fixture(name="formatter")
+def fixture_formatter() -> UniversalPlaintextFormatter:
+    """Create a basic formatter instance for testing."""
+    return UniversalPlaintextFormatter(
+        fmt="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    )
+
+
+@pytest.fixture(name="log_record")
+def fixture_log_record() -> logging.LogRecord:
+    """Create a basic log record for testing."""
+    return logging.LogRecord(
+        name="test_logger",
+        level=logging.INFO,
+        pathname="/path/to/file.py",
+        lineno=42,
+        msg="Test message",
+        args=(),
+        exc_info=None,
+    )
+
+
+def test_basic_formatting_without_extras(
+    formatter: UniversalPlaintextFormatter, log_record: logging.LogRecord
+) -> None:
+    """Test that basic formatting works without extra attributes."""
+    formatted = formatter.format(log_record)
+    assert "Test message" in formatted
+    assert "test_logger" in formatted
+    assert "INFO" in formatted
+
+
+def test_standard_attributes_excluded(
+    formatter: UniversalPlaintextFormatter, log_record: logging.LogRecord
+) -> None:
+    """Test that standard LogRecord attributes are excluded from extras."""
+    formatted = formatter.format(log_record)
+    # Standard attributes should not appear as extras
+    assert "pathname:" not in formatted
+    assert "lineno:" not in formatted
+    assert "levelname:" not in formatted
+
+
+def test_extra_attributes_included(
+    formatter: UniversalPlaintextFormatter, log_record: logging.LogRecord
+) -> None:
+    """Test that extra attributes are included in the formatted output."""
+    log_record.user_id = "12345"
+    log_record.request_id = "abc-def-ghi"
+
+    formatted = formatter.format(log_record)
+
+    assert "user_id: 12345" in formatted
+    assert "request_id: abc-def-ghi" in formatted
+
+
+def test_default_exclude_keys(
+    formatter: UniversalPlaintextFormatter, log_record: logging.LogRecord
+) -> None:
+    """Test that DEFAULT_EXCLUDE_KEYS (message, asctime) are excluded."""
+    # Add 'message' and 'asctime' as extra attributes (shouldn't appear in extras)
+    log_record.message = "This should be excluded"
+    log_record.asctime = "2025-01-01 12:00:00"
+
+    formatted = formatter.format(log_record)
+
+    # These should not appear as extras
+    lines = formatted.split("\n")
+    extra_lines = [
+        line for line in lines if line.strip().startswith(("message:", "asctime:"))
+    ]
+    assert len(extra_lines) == 0
+
+
+def test_multiple_extras_formatting(
+    formatter: UniversalPlaintextFormatter, log_record: logging.LogRecord
+) -> None:
+    """Test formatting with multiple extra attributes."""
+    log_record.user_id = "12345"
+    log_record.session_id = "session-xyz"
+    log_record.ip_address = "192.168.1.1"
+
+    formatted = formatter.format(log_record)
+
+    assert "user_id: 12345" in formatted
+    assert "session_id: session-xyz" in formatted
+    assert "ip_address: 192.168.1.1" in formatted
+
+    # Check that extras are indented
+    lines = formatted.split("\n")
+    extra_lines = [
+        line
+        for line in lines
+        if any(key in line for key in ("user_id:", "session_id:", "ip_address:"))
+    ]
+    for line in extra_lines:
+        assert line.startswith("    ")
+
+
+def test_config_exclude_keys_from_file(log_record: logging.LogRecord) -> None:
+    """Test that exclude_keys from configuration file are properly excluded."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        config_path = Path(tmpdir) / "logging.conf"
+        config_content = """[formatter_plaintextFormatter]
+exclude_keys = custom_field, another_field
+"""
+        config_path.write_text(config_content)
+
+        # Change to temp directory to read config
+        original_dir = os.getcwd()
+        try:
+            os.chdir(tmpdir)
+            formatter = UniversalPlaintextFormatter(fmt="%(message)s")
+
+            # Add attributes that should be excluded
+            log_record.custom_field = "should be excluded"
+            log_record.another_field = "also excluded"
+            log_record.included_field = "should be included"
+
+            formatted = formatter.format(log_record)
+
+            assert "custom_field:" not in formatted
+            assert "another_field:" not in formatted
+            assert "included_field: should be included" in formatted
+        finally:
+            os.chdir(original_dir)
+
+
+def test_config_exclude_keys_with_whitespace(
+    log_record: logging.LogRecord,
+) -> None:
+    """Test that whitespace in exclude_keys configuration is handled correctly."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        config_path = Path(tmpdir) / "logging.conf"
+        config_content = """[formatter_plaintextFormatter]
+exclude_keys = field1 ,  field2  , field3
+"""
+        config_path.write_text(config_content)
+
+        original_dir = os.getcwd()
+        try:
+            os.chdir(tmpdir)
+            formatter = UniversalPlaintextFormatter(fmt="%(message)s")
+
+            log_record.field1 = "excluded"
+            log_record.field2 = "excluded"
+            log_record.field3 = "excluded"
+
+            formatted = formatter.format(log_record)
+
+            assert "field1:" not in formatted
+            assert "field2:" not in formatted
+            assert "field3:" not in formatted
+        finally:
+            os.chdir(original_dir)
+
+
+def test_no_config_file(
+    formatter: UniversalPlaintextFormatter, log_record: logging.LogRecord
+) -> None:
+    """Test that formatter works correctly when config file doesn't exist."""
+    log_record.some_extra = "value"
+    formatted = formatter.format(log_record)
+
+    # Should still format correctly
+    assert "Test message" in formatted
+    assert "some_extra: value" in formatted
+
+
+def test_empty_extras(
+    formatter: UniversalPlaintextFormatter, log_record: logging.LogRecord
+) -> None:
+    """Test formatting when there are no extra attributes."""
+    formatted = formatter.format(log_record)
+
+    # Should only contain the base formatted message without extra newlines
+    lines = formatted.split("\n")
+    assert len([line for line in lines if line.strip()]) == 1
+
+
+def test_standard_attrs_caching(
+    formatter: UniversalPlaintextFormatter,
+) -> None:
+    """Test that standard attributes are cached after first call."""
+    assert formatter._standard_attrs is None
+
+    # First call should set the cache
+    standard_attrs = formatter._get_standard_attrs()
+    assert formatter._standard_attrs is not None
+    assert formatter._standard_attrs == standard_attrs
+
+    # Second call should return cached value
+    standard_attrs_2 = formatter._get_standard_attrs()
+    assert standard_attrs_2 is standard_attrs  # Same object
+
+
+def test_formatter_with_custom_format_string(
+    log_record: logging.LogRecord,
+) -> None:
+    """Test formatter with a custom format string."""
+    formatter = UniversalPlaintextFormatter(fmt="[%(levelname)s] %(message)s")
+    log_record.extra_data = "test"
+
+    formatted = formatter.format(log_record)
+
+    assert "[INFO] Test message" in formatted
+    assert "extra_data: test" in formatted
+
+
+def test_exclude_keys_combination(log_record: logging.LogRecord) -> None:
+    """Test that all exclusion sources are combined correctly."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        config_path = Path(tmpdir) / "logging.conf"
+        config_content = """[formatter_plaintextFormatter]
+exclude_keys = config_excluded
+"""
+        config_path.write_text(config_content)
+
+        original_dir = os.getcwd()
+        try:
+            os.chdir(tmpdir)
+            formatter = UniversalPlaintextFormatter(fmt="%(message)s")
+
+            # Add various attributes
+            log_record.pathname = "standard_attr"  # Standard LogRecord attribute
+            log_record.message = "default_excluded"  # DEFAULT_EXCLUDE_KEYS
+            log_record.config_excluded = "from_config"  # From config file
+            log_record.should_appear = "yes"  # Should appear
+
+            formatted = formatter.format(log_record)
+
+            # Only should_appear should be in extras
+            assert "pathname:" not in formatted  # Standard attribute
+            assert "message:" not in formatted  # DEFAULT_EXCLUDE_KEYS
+            assert "config_excluded:" not in formatted  # Config exclude
+            assert "should_appear: yes" in formatted  # Should be included
+        finally:
+            os.chdir(original_dir)

cledar/logging/universal_plaintext_formatter.py

@@ -0,0 +1,94 @@
+import configparser
+import logging
+from typing import Any
+
+
+class UniversalPlaintextFormatter(logging.Formatter):
+    """
+    A custom formatter for logging that extends the standard logging.Formatter.
+
+    This formatter adds the ability to include extra attributes from log records while
+    excluding standard attributes and configurable keys.
+    """
+
+    # Predefined exclusions - keys that should always be excluded
+    DEFAULT_EXCLUDE_KEYS = {"message", "asctime"}
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        """
+        Initialize the formatter with standard formatter parameters.
+
+        Args:
+            *args: Variable length argument list for the parent class.
+            **kwargs: Arbitrary keyword arguments for the parent class.
+        """
+        super().__init__(*args, **kwargs)
+        self._standard_attrs: set[str] | None = None
+        self._config_exclude_keys = self._load_exclude_keys_from_config()
+
+    def _load_exclude_keys_from_config(self) -> set[str]:
+        """
+        Load additional keys to exclude from the configuration file.
+
+        Returns:
+            set: A set of keys to exclude from log records.
+        """
+        try:
+            config = configparser.ConfigParser()
+            config.read("logging.conf")
+            if config.has_option("formatter_plaintextFormatter", "exclude_keys"):
+                exclude_str = config.get("formatter_plaintextFormatter", "exclude_keys")
+                return set(key.strip() for key in exclude_str.split(",") if key.strip())
+        except (configparser.Error, FileNotFoundError, PermissionError, ValueError):
+            pass
+        return set()
+
+    def _get_standard_attrs(self) -> set[str]:
+        """
+        Get the set of standard attributes to exclude from log records.
+
+        This includes standard LogRecord attributes, predefined exclusions,
+        and exclusions from configuration.
+
+        Returns:
+            set: A set of attribute names to exclude.
+        """
+        if self._standard_attrs is None:
+            dummy_record = logging.LogRecord(
+                name="dummy",
+                level=logging.INFO,
+                pathname="",
+                lineno=0,
+                msg="",
+                args=(),
+                exc_info=None,
+            )
+            # Combine standard attributes + predefined + from configuration
+            all_excludes = (
+                set(dummy_record.__dict__.keys())
+                | self.DEFAULT_EXCLUDE_KEYS
+                | self._config_exclude_keys
+            )
+            self._standard_attrs = all_excludes
+        return self._standard_attrs
+
+    def format(self, record: logging.LogRecord) -> str:
+        """
+        Format the log record, adding any extra attributes not in the standard set.
+
+        Args:
+            record: The log record to format.
+
+        Returns:
+            str: The formatted log message with extra attributes appended.
+        """
+        base = super().format(record)
+        extras = {
+            k: v
+            for k, v in record.__dict__.items()
+            if k not in self._get_standard_attrs()
+        }
+        if extras:
+            extras_str = "\n".join(f"    {k}: {v}" for k, v in extras.items())
+            return f"{base}\n{extras_str}"
+        return base

cledar/monitoring/README.md

@@ -0,0 +1,71 @@
+# Monitoring Service
+
+Monitoring service provides endpoints for healthchecks and Prometheus metrics.
+
+This module creates a monitoring server with uvicorn endpoints for managing health of applications.
+
+## Endpoints
+
+- GET /healthz/liveness ->
+Provides information if the app is alive. 
+Example return:
+```json
+{"status": "ok", "checks": {}}
+```
+- GET /healthz/readiness -> 
+Provides information if the app is ready and which components are active. 
+Example return:
+```json
+{"status": "ok", "checks": {"kafka_alive": true, "model_ready": true, "redis_alive": true}}
+```
+- GET /metrics ->
+Provides metrics collected by Prometheus client, to be used in metrics visualization client f.e. Grafana.
+
+## Usage
+
+In your app you have to define readiness checks - services that need to be running for app to work properly
+f.e. S3, Kafka, model loading, etc.
+This is usually solved by creating MonitoringContext object. 
+
+```python
+class MonitoringContext:
+    def __init__(self) -> None:
+        self.kafka_client: Optional[BaseKafkaClient] = None
+        self.redis_client: Optional[RedisService] = None
+        self._model_ready_flag: bool = False
+
+    def prepare_readiness_checks(self) -> dict[str, Callable[[], bool]]:
+        return {
+            "kafka_alive": self._kafka_check,
+            "model_ready": lambda: self._model_ready_flag,
+            "redis_alive": self._redis_alive,
+        }
+
+    def _redis_alive(self) -> bool:
+        if self.redis_client is None:
+            return False
+        return self.redis_client.is_alive()
+
+    def _kafka_check(self) -> bool:
+        if self.kafka_client is None:
+            return False
+        return self.kafka_client.is_alive()
+
+    def set_model_ready_flag(self, flag: bool) -> None:
+        self._model_ready_flag = flag
+```
+
+Now you can prepare your monitoring server by running in __main__:
+
+```python
+monitoring_context = MonitoringContext()
+monitoring_config = MonitoringServerConfig(
+    monitoring_context.prepare_readiness_checks()
+)
+monitoring_server = MonitoringServer(
+    host="0.0.0.0",
+    port=8000,
+    config=monitoring_config,
+)
+monitoring_server.start_monitoring_server()
+```

cledar/monitoring/__init__.py

@@ -0,0 +1,3 @@
+from .monitoring_server import EndpointFilter, MonitoringServer, MonitoringServerConfig
+
+__all__ = ["MonitoringServer", "MonitoringServerConfig", "EndpointFilter"]

cledar/monitoring/monitoring_server.py

@@ -0,0 +1,112 @@
+import json
+import logging
+import logging.config
+import threading
+from collections.abc import Callable
+
+import prometheus_client
+import uvicorn
+from fastapi import FastAPI, Response
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic.dataclasses import dataclass
+
+
+def _create_app() -> FastAPI:
+    app = FastAPI()
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["*"],
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+    return app
+
+
+def _run_monitoring_server(host: str, port: int, app: FastAPI) -> None:
+    uvicorn.run(app, host=host, port=port)
+
+
+@dataclass
+class MonitoringServerConfig:
+    readiness_checks: dict[str, Callable[[], bool]]
+    liveness_checks: dict[str, Callable[[], bool]] | None = None
+
+
+class EndpointFilter(logging.Filter):
+    def __init__(self, paths_excluded_for_logging: list[str]):
+        super().__init__()
+        self.paths_excluded_for_logging = paths_excluded_for_logging
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        return not any(
+            path in record.getMessage() for path in self.paths_excluded_for_logging
+        )
+
+
+class MonitoringServer:
+    PATHS_EXCLUDED_FOR_LOGGING = ["/healthz/readiness", "/healthz/liveness"]
+
+    def __init__(
+        self,
+        host: str,
+        port: int,
+        config: MonitoringServerConfig,
+    ):
+        self.config = config
+        self.host = host
+        self.port = port
+        logging.getLogger("uvicorn.access").addFilter(
+            EndpointFilter(self.PATHS_EXCLUDED_FOR_LOGGING)
+        )
+
+    def add_paths(self, app: FastAPI) -> None:
+        @app.get("/metrics")
+        async def get_metrics() -> Response:
+            return Response(
+                content=prometheus_client.generate_latest(),
+                media_type=prometheus_client.CONTENT_TYPE_LATEST,
+            )
+
+        @app.get("/healthz/liveness")
+        async def get_healthz_liveness() -> Response:
+            return await self._get_healthz_response(self.config.liveness_checks)
+
+        @app.get("/healthz/readiness")
+        async def get_healthz_readiness() -> Response:
+            return await self._get_healthz_response(self.config.readiness_checks)
+
+    async def _get_healthz_response(
+        self, checks: dict[str, Callable[[], bool]] | None
+    ) -> Response:
+        try:
+            results = (
+                {check_name: check_fn() for check_name, check_fn in checks.items()}
+                if checks
+                else {}
+            )
+
+            status = "error"
+            status_code = 503
+            if all(results.values()):
+                status = "ok"
+                status_code = 200
+
+            data = {"status": status, "checks": results}
+            data_json = json.dumps(data)
+            return Response(content=data_json, status_code=status_code)
+
+        except Exception as e:
+            data = {"status": "error", "message": str(e)}
+            data_json = json.dumps(data)
+            return Response(content=data_json, status_code=503)
+
+    def start_monitoring_server(self) -> None:
+        local_app = _create_app()
+        self.add_paths(local_app)
+        server_thread = threading.Thread(
+            target=_run_monitoring_server,
+            args=(self.host, self.port, local_app),
+        )
+        server_thread.daemon = True  # to ensure it dies with the main thread
+        server_thread.start()
+        logging.info("Monitoring server listening at %s:%s.", self.host, self.port)

cledar/monitoring/tests/integration/test_monitoring_server_int.py

@@ -0,0 +1,162 @@
+# pylint: disable=redefined-outer-name, unreachable
+import logging
+from collections.abc import Awaitable, Callable
+
+import pytest
+from fastapi import FastAPI
+from httpx import ASGITransport, AsyncClient
+
+from cledar.monitoring import EndpointFilter, MonitoringServer, MonitoringServerConfig
+
+# ------------------------------------------------------------------------------
+#  Fixtures
+# ------------------------------------------------------------------------------
+
+
+@pytest.fixture
+def test_app() -> FastAPI:
+    """Creates a FastAPI app with configured monitoring endpoints."""
+    app = FastAPI()
+    config = MonitoringServerConfig(
+        readiness_checks={"db": lambda: True},
+        liveness_checks={"heartbeat": lambda: True},
+    )
+    server = MonitoringServer("127.0.0.1", 8000, config)
+    server.add_paths(app)
+    return app
+
+
+ASGIClientFactory = Callable[[FastAPI], Awaitable[AsyncClient]]
+
+
+@pytest.fixture
+def asgi_client_factory() -> ASGIClientFactory:
+    """Factory to create an httpx.AsyncClient for ASGI apps (httpx>=0.28)."""
+
+    async def _make_client(app: FastAPI) -> AsyncClient:
+        transport = ASGITransport(app=app)
+        return AsyncClient(transport=transport, base_url="http://test")
+
+    return _make_client
+
+
+# ------------------------------------------------------------------------------
+#  Readiness & Liveness Endpoints
+# ------------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_readiness_ok(
+    test_app: FastAPI, asgi_client_factory: ASGIClientFactory
+) -> None:
+    """Readiness endpoint returns OK when all checks pass."""
+    async with await asgi_client_factory(test_app) as client:
+        response = await client.get("/healthz/readiness")
+
+    data = response.json()
+    assert response.status_code == 200
+    assert data["status"] == "ok"
+    assert data["checks"]["db"] is True
+
+
+@pytest.mark.asyncio
+async def test_liveness_ok(
+    test_app: FastAPI, asgi_client_factory: ASGIClientFactory
+) -> None:
+    """Liveness endpoint returns OK when all checks pass."""
+    async with await asgi_client_factory(test_app) as client:
+        response = await client.get("/healthz/liveness")
+
+    data = response.json()
+    assert response.status_code == 200
+    assert data["status"] == "ok"
+    assert data["checks"]["heartbeat"] is True
+
+
+@pytest.mark.asyncio
+async def test_readiness_failing_check(asgi_client_factory: ASGIClientFactory) -> None:
+    """Readiness endpoint returns 503 and error status if a check fails."""
+    app = FastAPI()
+    config = MonitoringServerConfig(readiness_checks={"db": lambda: False})
+    server = MonitoringServer("127.0.0.1", 8000, config)
+    server.add_paths(app)
+
+    async with await asgi_client_factory(app) as client:
+        response = await client.get("/healthz/readiness")
+
+    data = response.json()
+    assert response.status_code == 503
+    assert data["status"] == "error"
+    assert data["checks"]["db"] is False
+
+
+@pytest.mark.asyncio
+async def test_readiness_with_exception(asgi_client_factory: ASGIClientFactory) -> None:
+    """Readiness returns 503 if a check raises an exception."""
+
+    def failing_check() -> bool:
+        raise RuntimeError("DB not reachable")
+        return False
+
+    app = FastAPI()
+    config = MonitoringServerConfig(readiness_checks={"db": failing_check})
+    server = MonitoringServer("127.0.0.1", 8000, config)
+    server.add_paths(app)
+
+    async with await asgi_client_factory(app) as client:
+        response = await client.get("/healthz/readiness")
+
+    data = response.json()
+    assert response.status_code == 503
+    assert data["status"] == "error"
+    assert "DB not reachable" in data["message"]
+
+
+# ------------------------------------------------------------------------------
+#  Metrics Endpoint
+# ------------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_metrics_endpoint_returns_prometheus_format(
+    test_app: FastAPI, asgi_client_factory: ASGIClientFactory
+) -> None:
+    """Metrics endpoint should return valid Prometheus output."""
+    async with await asgi_client_factory(test_app) as client:
+        response = await client.get("/metrics")
+
+    assert response.status_code == 200
+    assert response.headers["content-type"].startswith("text/plain")
+    assert "python_info" in response.text  # standard metric
+
+
+# ------------------------------------------------------------------------------
+#  Logging Filter
+# ------------------------------------------------------------------------------
+
+
+def test_endpoint_filter_excludes_health_paths() -> None:
+    """EndpointFilter should exclude healthz paths from logs."""
+    filter_ = EndpointFilter(["/healthz/readiness", "/healthz/liveness"])
+
+    record_excluded = logging.LogRecord(
+        name="uvicorn.access",
+        level=logging.INFO,
+        pathname="",
+        lineno=0,
+        msg="GET /healthz/readiness 200 OK",
+        args=(),
+        exc_info=None,
+    )
+    assert filter_.filter(record_excluded) is False
+
+    record_included = logging.LogRecord(
+        name="uvicorn.access",
+        level=logging.INFO,
+        pathname="",
+        lineno=0,
+        msg="GET /metrics 200 OK",
+        args=(),
+        exc_info=None,
+    )
+    assert filter_.filter(record_included) is True