PyPI - apitally - Versions diffs - 0.13.0__py3-none-any.whl → 0.14.0rc1__py3-none-any.whl - Mend

apitally 0.13.0py3-none-any.whl → 0.14.0rc1py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (22) hide show

apitally/client/{asyncio.py → client_asyncio.py} +55 -16
apitally/client/client_base.py +97 -0
apitally/client/{threading.py → client_threading.py} +51 -10
apitally/client/consumers.py +66 -0
apitally/client/request_logging.py +344 -0
apitally/client/requests.py +86 -0
apitally/client/server_errors.py +126 -0
apitally/client/validation_errors.py +58 -0
apitally/common.py +10 -1
apitally/django.py +112 -46
apitally/django_ninja.py +2 -2
apitally/django_rest_framework.py +2 -2
apitally/fastapi.py +2 -2
apitally/flask.py +100 -26
apitally/litestar.py +122 -54
apitally/starlette.py +90 -29
{apitally-0.13.0.dist-info → apitally-0.14.0rc1.dist-info}/METADATA +1 -2
apitally-0.14.0rc1.dist-info/RECORD +24 -0
{apitally-0.13.0.dist-info → apitally-0.14.0rc1.dist-info}/WHEEL +1 -1
apitally/client/base.py +0 -404
apitally-0.13.0.dist-info/RECORD +0 -19
{apitally-0.13.0.dist-info → apitally-0.14.0rc1.dist-info}/licenses/LICENSE +0 -0

apitally/client/request_logging.py ADDED Viewed

@@ -0,0 +1,344 @@
+import base64
+import gzip
+import re
+import tempfile
+import threading
+import time
+from collections import deque
+from contextlib import suppress
+from dataclasses import dataclass, field
+from functools import lru_cache
+from io import BufferedReader
+from pathlib import Path
+from typing import Any, AsyncIterator, Callable, Dict, List, Mapping, Optional, Tuple, TypedDict
+from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
+from uuid import uuid4
+from apitally.client.logging import get_logger
+logger = get_logger(__name__)
+MAX_BODY_SIZE = 50_000  # 50 KB (uncompressed)
+MAX_FILE_SIZE = 1_000_000  # 1 MB (compressed)
+MAX_REQUESTS_IN_DEQUE = 100  # Written to file every second, so limits logging to 100 rps
+MAX_FILES_IN_DEQUE = 50
+BODY_TOO_LARGE = b"<body too large>"
+BODY_MASKED = b"<masked>"
+MASKED = "******"
+ALLOWED_CONTENT_TYPES = ["application/json", "text/plain"]
+EXCLUDE_PATH_PATTERNS = [
+    r"/_?healthz?$",
+    r"/_?health[_-]?checks?$",
+    r"/_?heart[_-]?beats?$",
+    r"/ping$",
+    r"/ready$",
+    r"/live$",
+]
+MASK_QUERY_PARAM_PATTERNS = [
+    r"auth",
+    r"api-?key",
+    r"secret",
+    r"token",
+    r"password",
+    r"pwd",
+]
+MASK_HEADER_PATTERNS = [
+    r"auth",
+    r"api-?key",
+    r"secret",
+    r"token",
+    r"cookie",
+]
+class RequestDict(TypedDict):
+    timestamp: float
+    method: str
+    path: Optional[str]
+    url: str
+    headers: List[Tuple[str, str]]
+    size: Optional[int]
+    consumer: Optional[str]
+    body: Optional[bytes]
+class ResponseDict(TypedDict):
+    status_code: int
+    response_time: float
+    headers: List[Tuple[str, str]]
+    size: Optional[int]
+    body: Optional[bytes]
+@dataclass
+class RequestLoggingConfig:
+    """
+    Configuration for request logging.
+    Attributes:
+        enabled: Whether request logging is enabled
+        log_query_params: Whether to log query parameter values
+        log_request_headers: Whether to log request header values
+        log_request_body: Whether to log the request body (only if JSON or plain text)
+        log_response_headers: Whether to log response header values
+        log_response_body: Whether to log the response body (only if JSON or plain text)
+        mask_query_params: Query parameter names to mask in logs. Expects regular expressions.
+        mask_headers: Header names to mask in logs. Expects regular expressions.
+        mask_request_body_callback: Callback to mask the request body. Expects (method, path, body) and returns the masked body as bytes or None.
+        mask_response_body_callback: Callback to mask the response body. Expects (method, path, body) and returns the masked body as bytes or None.
+        exclude_paths: Paths to exclude from logging. Expects regular expressions.
+        exclude_callback: Callback to exclude requests from logging. Should expect two arguments, `request: RequestDict` and `response: ResponseDict`, and return True to exclude the request.
+    """
+    enabled: bool = False
+    log_query_params: bool = True
+    log_request_headers: bool = False
+    log_request_body: bool = False
+    log_response_headers: bool = True
+    log_response_body: bool = False
+    mask_query_params: List[str] = field(default_factory=list)
+    mask_headers: List[str] = field(default_factory=list)
+    mask_request_body_callback: Optional[Callable[[str, str, bytes], Optional[bytes]]] = None
+    mask_response_body_callback: Optional[Callable[[str, str, bytes], Optional[bytes]]] = None
+    exclude_paths: List[str] = field(default_factory=list)
+    exclude_callback: Optional[Callable[[RequestDict, ResponseDict], bool]] = None
+class TempGzipFile:
+    def __init__(self) -> None:
+        self.uuid = uuid4()
+        self.file = tempfile.NamedTemporaryFile(
+            suffix=".gz",
+            prefix="apitally-",
+            delete=False,
+        )
+        self.gzip_file = gzip.open(self.file, "wb")
+    @property
+    def path(self) -> Path:
+        return Path(self.file.name)
+    @property
+    def size(self) -> int:
+        return self.file.tell()
+    def write_line(self, data: bytes) -> None:
+        self.gzip_file.write(data + b"\n")
+    def open_compressed(self) -> BufferedReader:
+        return open(self.path, "rb")
+    async def stream_lines_compressed(self) -> AsyncIterator[bytes]:
+        with open(self.path, "rb") as fp:
+            for line in fp:
+                yield line
+    def close(self) -> None:
+        self.gzip_file.close()
+        self.file.close()
+    def delete(self) -> None:
+        self.close()
+        self.path.unlink(missing_ok=True)
+class RequestLogger:
+    def __init__(self, config: Optional[RequestLoggingConfig]) -> None:
+        self.config = config or RequestLoggingConfig()
+        self.enabled = self.config.enabled and _check_writable_fs()
+        self.serialize = _get_json_serializer()
+        self.write_deque: deque[bytes] = deque([], MAX_REQUESTS_IN_DEQUE)
+        self.file_deque: deque[TempGzipFile] = deque([])
+        self.file: Optional[TempGzipFile] = None
+        self.lock = threading.Lock()
+        self.suspend_until: Optional[float] = None
+    @property
+    def current_file_size(self) -> int:
+        return self.file.size if self.file is not None else 0
+    def log_request(self, request: RequestDict, response: ResponseDict) -> None:
+        if not self.enabled or self.suspend_until is not None:
+            return
+        parsed_url = urlparse(request["url"])
+        if self._should_exclude_path(request["path"] or parsed_url.path) or self._should_exclude(request, response):
+            return
+        query = self._mask_query_params(parsed_url.query) if self.config.log_query_params else ""
+        request["url"] = urlunparse(parsed_url._replace(query=query))
+        request["headers"] = self._mask_headers(request["headers"]) if self.config.log_request_headers else []
+        response["headers"] = self._mask_headers(response["headers"]) if self.config.log_response_headers else []
+        if not self.config.log_request_body or not self._has_supported_content_type(request["headers"]):
+            request["body"] = None
+        elif (
+            self.config.mask_request_body_callback is not None
+            and request["body"] is not None
+            and request["body"] != BODY_TOO_LARGE
+        ):
+            try:
+                request["body"] = self.config.mask_request_body_callback(
+                    request["method"], request["path"] or parsed_url.path, request["body"]
+                )
+            except Exception:  # pragma: no cover
+                logger.exception("User-provided mask_request_body_callback function raised an exception")
+                request["body"] = None
+            if request["body"] is None:
+                request["body"] = BODY_MASKED
+        if request["body"] is not None and len(request["body"]) > MAX_BODY_SIZE:
+            request["body"] = BODY_TOO_LARGE
+        if not self.config.log_response_body or not self._has_supported_content_type(response["headers"]):
+            response["body"] = None
+        elif (
+            self.config.mask_response_body_callback is not None
+            and response["body"] is not None
+            and response["body"] != BODY_TOO_LARGE
+        ):
+            try:
+                response["body"] = self.config.mask_response_body_callback(
+                    request["method"], request["path"] or parsed_url.path, response["body"]
+                )
+            except Exception:  # pragma: no cover
+                logger.exception("User-provided mask_response_body_callback function raised an exception")
+                response["body"] = None
+            if response["body"] is None:
+                response["body"] = BODY_MASKED
+        if response["body"] is not None and len(response["body"]) > MAX_BODY_SIZE:
+            response["body"] = BODY_TOO_LARGE
+        item = {
+            "uuid": str(uuid4()),
+            "request": _skip_empty_values(request),
+            "response": _skip_empty_values(response),
+        }
+        serialized_item = self.serialize(item)
+        self.write_deque.append(serialized_item)
+    def write_to_file(self) -> None:
+        if not self.enabled or len(self.write_deque) == 0:
+            return
+        with self.lock:
+            if self.file is None:
+                self.file = TempGzipFile()
+            while True:
+                try:
+                    item = self.write_deque.popleft()
+                    self.file.write_line(item)
+                except IndexError:
+                    break
+    def get_file(self) -> Optional[TempGzipFile]:
+        try:
+            return self.file_deque.popleft()
+        except IndexError:
+            return None
+    def retry_file_later(self, file: TempGzipFile) -> None:
+        self.file_deque.appendleft(file)
+    def rotate_file(self) -> None:
+        if self.file is not None:
+            with self.lock:
+                self.file.close()
+                self.file_deque.append(self.file)
+                self.file = None
+    def maintain(self) -> None:
+        if self.current_file_size > MAX_FILE_SIZE:
+            self.rotate_file()
+        while len(self.file_deque) > MAX_FILES_IN_DEQUE:
+            file = self.file_deque.popleft()
+            file.delete()
+        if self.suspend_until is not None and self.suspend_until < time.time():
+            self.suspend_until = None
+    def clear(self) -> None:
+        self.write_deque.clear()
+        self.rotate_file()
+        for file in self.file_deque:
+            file.delete()
+        self.file_deque.clear()
+    def close(self) -> None:
+        self.enabled = False
+        self.clear()
+    def _should_exclude(self, request: RequestDict, response: ResponseDict) -> bool:
+        if self.config.exclude_callback is not None:
+            return self.config.exclude_callback(request, response)
+        return False
+    @lru_cache(maxsize=1000)
+    def _should_exclude_path(self, url_path: str) -> bool:
+        patterns = self.config.exclude_paths + EXCLUDE_PATH_PATTERNS
+        return self._match_patterns(url_path, patterns)
+    def _mask_query_params(self, query: str) -> str:
+        query_params = parse_qsl(query)
+        masked_query_params = [(k, v if not self._should_mask_query_param(k) else MASKED) for k, v in query_params]
+        return urlencode(masked_query_params)
+    def _mask_headers(self, headers: List[Tuple[str, str]]) -> List[Tuple[str, str]]:
+        return [(k, v if not self._should_mask_header(k) else MASKED) for k, v in headers]
+    @lru_cache(maxsize=100)
+    def _should_mask_query_param(self, query_param_name: str) -> bool:
+        patterns = self.config.mask_query_params + MASK_QUERY_PARAM_PATTERNS
+        return self._match_patterns(query_param_name, patterns)
+    @lru_cache(maxsize=100)
+    def _should_mask_header(self, header_name: str) -> bool:
+        patterns = self.config.mask_headers + MASK_HEADER_PATTERNS
+        return self._match_patterns(header_name, patterns)
+    @staticmethod
+    def _match_patterns(value: str, patterns: List[str]) -> bool:
+        for pattern in patterns:
+            with suppress(re.error):
+                if re.search(pattern, value, re.I) is not None:
+                    return True
+        return False
+    @staticmethod
+    def _has_supported_content_type(headers: List[Tuple[str, str]]) -> bool:
+        content_type = next((v for k, v in headers if k.lower() == "content-type"), None)
+        return content_type is not None and any(content_type.startswith(t) for t in ALLOWED_CONTENT_TYPES)
+def _check_writable_fs() -> bool:
+    try:
+        with tempfile.NamedTemporaryFile():
+            return True
+    except (IOError, OSError):  # pragma: no cover
+        logger.error("Unable to create temporary file for request logging")
+        return False
+def _get_json_serializer() -> Callable[[Any], bytes]:
+    def default(obj: Any) -> Any:
+        if isinstance(obj, bytes):
+            return base64.b64encode(obj).decode()
+        raise TypeError  # pragma: no cover
+    try:
+        import orjson  # type: ignore
+        def orjson_dumps(obj: Any) -> bytes:
+            return orjson.dumps(obj, default=default)
+        return orjson_dumps
+    except ImportError:
+        import json
+        def json_dumps(obj: Any) -> bytes:
+            return json.dumps(obj, separators=(",", ":"), default=default).encode()
+        return json_dumps
+def _skip_empty_values(data: Mapping) -> Dict:
+    return {
+        k: v for k, v in data.items() if v is not None and not (isinstance(v, (list, dict, bytes, str)) and len(v) == 0)
+    }

apitally/client/requests.py ADDED Viewed

@@ -0,0 +1,86 @@
+from __future__ import annotations
+import contextlib
+import threading
+from collections import Counter
+from dataclasses import dataclass
+from math import floor
+from typing import Any, Dict, List, Optional
+@dataclass(frozen=True)
+class RequestInfo:
+    consumer: Optional[str]
+    method: str
+    path: str
+    status_code: int
+class RequestCounter:
+    def __init__(self) -> None:
+        self.request_counts: Counter[RequestInfo] = Counter()
+        self.request_size_sums: Counter[RequestInfo] = Counter()
+        self.response_size_sums: Counter[RequestInfo] = Counter()
+        self.response_times: Dict[RequestInfo, Counter[int]] = {}
+        self.request_sizes: Dict[RequestInfo, Counter[int]] = {}
+        self.response_sizes: Dict[RequestInfo, Counter[int]] = {}
+        self._lock = threading.Lock()
+    def add_request(
+        self,
+        consumer: Optional[str],
+        method: str,
+        path: str,
+        status_code: int,
+        response_time: float,
+        request_size: str | int | None = None,
+        response_size: str | int | None = None,
+    ) -> None:
+        request_info = RequestInfo(
+            consumer=consumer,
+            method=method.upper(),
+            path=path,
+            status_code=status_code,
+        )
+        response_time_ms_bin = int(floor(response_time / 0.01) * 10)  # In ms, rounded down to nearest 10ms
+        with self._lock:
+            self.request_counts[request_info] += 1
+            self.response_times.setdefault(request_info, Counter())[response_time_ms_bin] += 1
+            if request_size is not None:
+                with contextlib.suppress(ValueError):
+                    request_size = int(request_size)
+                    request_size_kb_bin = request_size // 1000  # In KB, rounded down to nearest 1KB
+                    self.request_size_sums[request_info] += request_size
+                    self.request_sizes.setdefault(request_info, Counter())[request_size_kb_bin] += 1
+            if response_size is not None:
+                with contextlib.suppress(ValueError):
+                    response_size = int(response_size)
+                    response_size_kb_bin = response_size // 1000  # In KB, rounded down to nearest 1KB
+                    self.response_size_sums[request_info] += response_size
+                    self.response_sizes.setdefault(request_info, Counter())[response_size_kb_bin] += 1
+    def get_and_reset_requests(self) -> List[Dict[str, Any]]:
+        data: List[Dict[str, Any]] = []
+        with self._lock:
+            for request_info, count in self.request_counts.items():
+                data.append(
+                    {
+                        "consumer": request_info.consumer,
+                        "method": request_info.method,
+                        "path": request_info.path,
+                        "status_code": request_info.status_code,
+                        "request_count": count,
+                        "request_size_sum": self.request_size_sums.get(request_info, 0),
+                        "response_size_sum": self.response_size_sums.get(request_info, 0),
+                        "response_times": self.response_times.get(request_info) or Counter(),
+                        "request_sizes": self.request_sizes.get(request_info) or Counter(),
+                        "response_sizes": self.response_sizes.get(request_info) or Counter(),
+                    }
+                )
+            self.request_counts.clear()
+            self.request_size_sums.clear()
+            self.response_size_sums.clear()
+            self.response_times.clear()
+            self.request_sizes.clear()
+            self.response_sizes.clear()
+        return data

apitally/client/server_errors.py ADDED Viewed

@@ -0,0 +1,126 @@
+from __future__ import annotations
+import asyncio
+import contextlib
+import sys
+import threading
+import traceback
+from collections import Counter
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Set
+MAX_EXCEPTION_MSG_LENGTH = 2048
+MAX_EXCEPTION_TRACEBACK_LENGTH = 65536
+@dataclass(frozen=True)
+class ServerError:
+    consumer: Optional[str]
+    method: str
+    path: str
+    type: str
+    msg: str
+    traceback: str
+class ServerErrorCounter:
+    def __init__(self) -> None:
+        self.error_counts: Counter[ServerError] = Counter()
+        self.sentry_event_ids: Dict[ServerError, str] = {}
+        self._lock = threading.Lock()
+        self._tasks: Set[asyncio.Task] = set()
+    def add_server_error(self, consumer: Optional[str], method: str, path: str, exception: BaseException) -> None:
+        if not isinstance(exception, BaseException):
+            return  # pragma: no cover
+        exception_type = type(exception)
+        with self._lock:
+            server_error = ServerError(
+                consumer=consumer,
+                method=method.upper(),
+                path=path,
+                type=f"{exception_type.__module__}.{exception_type.__qualname__}",
+                msg=self._get_truncated_exception_msg(exception),
+                traceback=self._get_truncated_exception_traceback(exception),
+            )
+            self.error_counts[server_error] += 1
+            self.capture_sentry_event_id(server_error)
+    def capture_sentry_event_id(self, server_error: ServerError) -> None:
+        try:
+            from sentry_sdk.hub import Hub
+            from sentry_sdk.scope import Scope
+        except ImportError:
+            return  # pragma: no cover
+        if not hasattr(Scope, "get_isolation_scope") or not hasattr(Scope, "_last_event_id"):
+            # sentry-sdk < 2.2.0 is not supported
+            return  # pragma: no cover
+        if Hub.current.client is None:
+            return  # sentry-sdk not initialized
+        scope = Scope.get_isolation_scope()
+        if event_id := scope._last_event_id:
+            self.sentry_event_ids[server_error] = event_id
+            return
+        async def _wait_for_sentry_event_id(scope: Scope) -> None:
+            i = 0
+            while not (event_id := scope._last_event_id) and i < 100:
+                i += 1
+                await asyncio.sleep(0.001)
+            if event_id:
+                self.sentry_event_ids[server_error] = event_id
+        with contextlib.suppress(RuntimeError):  # ignore no running loop
+            loop = asyncio.get_running_loop()
+            task = loop.create_task(_wait_for_sentry_event_id(scope))
+            self._tasks.add(task)
+            task.add_done_callback(self._tasks.discard)
+    def get_and_reset_server_errors(self) -> List[Dict[str, Any]]:
+        data: List[Dict[str, Any]] = []
+        with self._lock:
+            for server_error, count in self.error_counts.items():
+                data.append(
+                    {
+                        "consumer": server_error.consumer,
+                        "method": server_error.method,
+                        "path": server_error.path,
+                        "type": server_error.type,
+                        "msg": server_error.msg,
+                        "traceback": server_error.traceback,
+                        "sentry_event_id": self.sentry_event_ids.get(server_error),
+                        "error_count": count,
+                    }
+                )
+            self.error_counts.clear()
+            self.sentry_event_ids.clear()
+        return data
+    @staticmethod
+    def _get_truncated_exception_msg(exception: BaseException) -> str:
+        msg = str(exception).strip()
+        if len(msg) <= MAX_EXCEPTION_MSG_LENGTH:
+            return msg
+        suffix = "... (truncated)"
+        cutoff = MAX_EXCEPTION_MSG_LENGTH - len(suffix)
+        return msg[:cutoff] + suffix
+    @staticmethod
+    def _get_truncated_exception_traceback(exception: BaseException) -> str:
+        prefix = "... (truncated) ...\n"
+        cutoff = MAX_EXCEPTION_TRACEBACK_LENGTH - len(prefix)
+        lines = []
+        length = 0
+        if sys.version_info >= (3, 10):
+            traceback_lines = traceback.format_exception(exception)
+        else:
+            traceback_lines = traceback.format_exception(type(exception), exception, exception.__traceback__)
+        for line in traceback_lines[::-1]:
+            if length + len(line) > cutoff:
+                lines.append(prefix)
+                break
+            lines.append(line)
+            length += len(line)
+        return "".join(lines[::-1]).strip()

apitally/client/validation_errors.py ADDED Viewed

@@ -0,0 +1,58 @@
+from __future__ import annotations
+import threading
+from collections import Counter
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Tuple
+@dataclass(frozen=True)
+class ValidationError:
+    consumer: Optional[str]
+    method: str
+    path: str
+    loc: Tuple[str, ...]
+    msg: str
+    type: str
+class ValidationErrorCounter:
+    def __init__(self) -> None:
+        self.error_counts: Counter[ValidationError] = Counter()
+        self._lock = threading.Lock()
+    def add_validation_errors(
+        self, consumer: Optional[str], method: str, path: str, detail: List[Dict[str, Any]]
+    ) -> None:
+        with self._lock:
+            for error in detail:
+                try:
+                    validation_error = ValidationError(
+                        consumer=consumer,
+                        method=method.upper(),
+                        path=path,
+                        loc=tuple(str(loc) for loc in error["loc"]),
+                        msg=error["msg"],
+                        type=error["type"],
+                    )
+                    self.error_counts[validation_error] += 1
+                except (KeyError, TypeError):  # pragma: no cover
+                    pass
+    def get_and_reset_validation_errors(self) -> List[Dict[str, Any]]:
+        data: List[Dict[str, Any]] = []
+        with self._lock:
+            for validation_error, count in self.error_counts.items():
+                data.append(
+                    {
+                        "consumer": validation_error.consumer,
+                        "method": validation_error.method,
+                        "path": validation_error.path,
+                        "loc": validation_error.loc,
+                        "msg": validation_error.msg,
+                        "type": validation_error.type,
+                        "error_count": count,
+                    }
+                )
+            self.error_counts.clear()
+        return data

apitally/common.py CHANGED Viewed

@@ -1,6 +1,15 @@
 import sys
 from importlib.metadata import PackageNotFoundError, version
-from typing import Dict, Optional
+from typing import Dict, Optional, Union
+def parse_int(x: Union[str, int, None]) -> Optional[int]:
+    if x is None:
+        return None
+    try:
+        return int(x)
+    except ValueError:
+        return None
 def get_versions(*packages, app_version: Optional[str] = None) -> Dict[str, str]:

apitally 0.13.0__py3-none-any.whl → 0.14.0rc1__py3-none-any.whl

apitally 0.13.0py3-none-any.whl → 0.14.0rc1py3-none-any.whl