apitally 0.13.0a1__py3-none-any.whl → 0.14.0rc1__py3-none-any.whl

apitally/django.py

@@ -13,10 +13,15 @@ from django.conf import settings
 from django.urls import URLPattern, URLResolver, get_resolver
 from django.utils.module_loading import import_string
 
-from apitally.client.base import Consumer as ApitallyConsumer
+from apitally.client.client_threading import ApitallyClient
+from apitally.client.consumers import Consumer as ApitallyConsumer
 from apitally.client.logging import get_logger
-from apitally.client.threading import ApitallyClient
-from apitally.common import get_versions
+from apitally.client.request_logging import (
+    BODY_TOO_LARGE,
+    MAX_BODY_SIZE,
+    RequestLoggingConfig,
+)
+from apitally.common import get_versions, parse_int
 
 
 if TYPE_CHECKING:
@@ -24,7 +29,7 @@ if TYPE_CHECKING:
     from ninja import NinjaAPI
 
 
-__all__ = ["ApitallyMiddleware", "ApitallyConsumer"]
+__all__ = ["ApitallyMiddleware", "ApitallyConsumer", "RequestLoggingConfig"]
 logger = get_logger(__name__)
 
 
@@ -32,6 +37,7 @@ logger = get_logger(__name__)
 class ApitallyMiddlewareConfig:
     client_id: str
     env: str
+    request_logging_config: Optional[RequestLoggingConfig]
     app_version: Optional[str]
     identify_consumer_callback: Optional[Callable[[HttpRequest], Union[str, ApitallyConsumer, None]]]
     urlconfs: List[Optional[str]]
@@ -61,7 +67,11 @@ class ApitallyMiddleware:
         if self.ninja_available and None not in self.config.urlconfs:
             self.callbacks.update(_get_ninja_callbacks(self.config.urlconfs))
 
-        self.client = ApitallyClient(client_id=self.config.client_id, env=self.config.env)
+        self.client = ApitallyClient(
+            client_id=self.config.client_id,
+            env=self.config.env,
+            request_logging_config=self.config.request_logging_config,
+        )
         self.client.start_sync_loop()
         self.client.set_startup_data(
             _get_startup_data(
@@ -70,11 +80,19 @@ class ApitallyMiddleware:
             )
         )
 
+        self.capture_request_body = (
+            self.client.request_logger.config.enabled and self.client.request_logger.config.log_request_body
+        )
+        self.capture_response_body = (
+            self.client.request_logger.config.enabled and self.client.request_logger.config.log_response_body
+        )
+
     @classmethod
     def configure(
         cls,
         client_id: str,
         env: str = "dev",
+        request_logging_config: Optional[RequestLoggingConfig] = None,
         app_version: Optional[str] = None,
         identify_consumer_callback: Optional[str] = None,
         urlconf: Optional[Union[List[Optional[str]], str]] = None,
@@ -82,6 +100,7 @@ class ApitallyMiddleware:
         cls.config = ApitallyMiddlewareConfig(
             client_id=client_id,
             env=env,
+            request_logging_config=request_logging_config,
             app_version=app_version,
             identify_consumer_callback=import_string(identify_consumer_callback)
             if identify_consumer_callback
@@ -90,11 +109,31 @@ class ApitallyMiddleware:
         )
 
     def __call__(self, request: HttpRequest) -> HttpResponse:
-        start_time = time.perf_counter()
-        response = self.get_response(request)
-        response_time = time.perf_counter() - start_time
-        path = self.get_path(request)
-        if request.method is not None and request.method != "OPTIONS" and path is not None:
+        if request.method is not None and request.method != "OPTIONS":
+            timestamp = time.time()
+            request_size = parse_int(request.headers.get("Content-Length"))
+            request_body = b""
+            if self.capture_request_body:
+                request_body = (
+                    request.body
+                    if request_size is not None and request_size <= MAX_BODY_SIZE and len(request.body) <= MAX_BODY_SIZE
+                    else BODY_TOO_LARGE
+                )
+
+            start_time = time.perf_counter()
+            response = self.get_response(request)
+            response_time = time.perf_counter() - start_time
+            response_size = (
+                parse_int(response["Content-Length"])
+                if response.has_header("Content-Length")
+                else (len(response.content) if not response.streaming else None)
+            )
+            response_body = b""
+            if self.capture_response_body and not response.streaming:
+                response_body = (
+                    response.content if response_size is not None and response_size <= MAX_BODY_SIZE else BODY_TOO_LARGE
+                )
+
             try:
                 consumer = self.get_consumer(request)
                 consumer_identifier = consumer.identifier if consumer else None
@@ -102,48 +141,75 @@ class ApitallyMiddleware:
             except Exception:  # pragma: no cover
                 logger.exception("Failed to get consumer for request")
                 consumer_identifier = None
-            try:
-                self.client.request_counter.add_request(
-                    consumer=consumer_identifier,
-                    method=request.method,
-                    path=path,
-                    status_code=response.status_code,
-                    response_time=response_time,
-                    request_size=request.headers.get("Content-Length"),
-                    response_size=response["Content-Length"]
-                    if response.has_header("Content-Length")
-                    else (len(response.content) if not response.streaming else None),
-                )
-            except Exception:  # pragma: no cover
-                logger.exception("Failed to log request metadata")
-            if (
-                response.status_code == 422
-                and (content_type := response.get("Content-Type")) is not None
-                and content_type.startswith("application/json")
-            ):
-                try:
-                    with contextlib.suppress(json.JSONDecodeError):
-                        body = json.loads(response.content)
-                        if isinstance(body, dict) and "detail" in body and isinstance(body["detail"], list):
-                            # Log Django Ninja / Pydantic validation errors
-                            self.client.validation_error_counter.add_validation_errors(
-                                consumer=consumer_identifier,
-                                method=request.method,
-                                path=path,
-                                detail=body["detail"],
-                            )
-                except Exception:  # pragma: no cover
-                    logger.exception("Failed to log validation errors")
-            if response.status_code == 500 and hasattr(request, "unhandled_exception"):
+
+            path = self.get_path(request)
+            if path is not None:
                 try:
-                    self.client.server_error_counter.add_server_error(
+                    self.client.request_counter.add_request(
                         consumer=consumer_identifier,
                         method=request.method,
                         path=path,
-                        exception=getattr(request, "unhandled_exception"),
+                        status_code=response.status_code,
+                        response_time=response_time,
+                        request_size=request_size,
+                        response_size=response_size,
                     )
                 except Exception:  # pragma: no cover
-                    logger.exception("Failed to log server error")
+                    logger.exception("Failed to log request metadata")
+
+                if (
+                    response.status_code == 422
+                    and (content_type := response.get("Content-Type")) is not None
+                    and content_type.startswith("application/json")
+                ):
+                    try:
+                        with contextlib.suppress(json.JSONDecodeError):
+                            body = json.loads(response.content)
+                            if isinstance(body, dict) and "detail" in body and isinstance(body["detail"], list):
+                                # Log Django Ninja / Pydantic validation errors
+                                self.client.validation_error_counter.add_validation_errors(
+                                    consumer=consumer_identifier,
+                                    method=request.method,
+                                    path=path,
+                                    detail=body["detail"],
+                                )
+                    except Exception:  # pragma: no cover
+                        logger.exception("Failed to log validation errors")
+
+                if response.status_code == 500 and hasattr(request, "unhandled_exception"):
+                    try:
+                        self.client.server_error_counter.add_server_error(
+                            consumer=consumer_identifier,
+                            method=request.method,
+                            path=path,
+                            exception=getattr(request, "unhandled_exception"),
+                        )
+                    except Exception:  # pragma: no cover
+                        logger.exception("Failed to log server error")
+
+            if self.client.request_logger.enabled:
+                self.client.request_logger.log_request(
+                    request={
+                        "timestamp": timestamp,
+                        "method": request.method,
+                        "path": path,
+                        "url": request.build_absolute_uri(),
+                        "headers": list(request.headers.items()),
+                        "size": request_size,
+                        "consumer": consumer_identifier,
+                        "body": request_body,
+                    },
+                    response={
+                        "status_code": response.status_code,
+                        "response_time": response_time,
+                        "headers": list(response.items()),
+                        "size": response_size,
+                        "body": response_body,
+                    },
+                )
+        else:
+            response = self.get_response(request)
+
         return response
 
     def process_exception(self, request: HttpRequest, exception: Exception) -> None:

apitally/django_ninja.py

@@ -1,4 +1,4 @@
-from apitally.django import ApitallyConsumer, ApitallyMiddleware
+from apitally.django import ApitallyConsumer, ApitallyMiddleware, RequestLoggingConfig
 
 
-__all__ = ["ApitallyMiddleware", "ApitallyConsumer"]
+__all__ = ["ApitallyMiddleware", "ApitallyConsumer", "RequestLoggingConfig"]

apitally/django_rest_framework.py

@@ -1,4 +1,4 @@
-from apitally.django import ApitallyConsumer, ApitallyMiddleware
+from apitally.django import ApitallyConsumer, ApitallyMiddleware, RequestLoggingConfig
 
 
-__all__ = ["ApitallyMiddleware", "ApitallyConsumer"]
+__all__ = ["ApitallyMiddleware", "ApitallyConsumer", "RequestLoggingConfig"]

apitally/fastapi.py

@@ -1,4 +1,4 @@
-from apitally.starlette import ApitallyConsumer, ApitallyMiddleware
+from apitally.starlette import ApitallyConsumer, ApitallyMiddleware, RequestLoggingConfig
 
 
-__all__ = ["ApitallyMiddleware", "ApitallyConsumer"]
+__all__ = ["ApitallyMiddleware", "ApitallyConsumer", "RequestLoggingConfig"]

apitally/flask.py

@@ -1,18 +1,24 @@
 from __future__ import annotations
 
 import time
+from io import BytesIO
 from threading import Timer
 from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple
 from warnings import warn
 
 from flask import Flask, g
-from flask.wrappers import Response
+from flask.wrappers import Request, Response
 from werkzeug.datastructures import Headers
 from werkzeug.exceptions import NotFound
 from werkzeug.test import Client
 
-from apitally.client.base import Consumer as ApitallyConsumer
-from apitally.client.threading import ApitallyClient
+from apitally.client.client_threading import ApitallyClient
+from apitally.client.consumers import Consumer as ApitallyConsumer
+from apitally.client.request_logging import (
+    BODY_TOO_LARGE,
+    MAX_BODY_SIZE,
+    RequestLoggingConfig,
+)
 from apitally.common import get_versions
 
 
@@ -21,7 +27,7 @@ if TYPE_CHECKING:
     from werkzeug.routing.map import Map
 
 
-__all__ = ["ApitallyMiddleware", "ApitallyConsumer"]
+__all__ = ["ApitallyMiddleware", "ApitallyConsumer", "RequestLoggingConfig"]
 
 
 class ApitallyMiddleware:
@@ -30,18 +36,24 @@ class ApitallyMiddleware:
         app: Flask,
         client_id: str,
         env: str = "dev",
+        request_logging_config: Optional[RequestLoggingConfig] = None,
         app_version: Optional[str] = None,
         openapi_url: Optional[str] = None,
-        filter_unhandled_paths: bool = True,
     ) -> None:
         self.app = app
         self.wsgi_app = app.wsgi_app
-        self.filter_unhandled_paths = filter_unhandled_paths
         self.patch_handle_exception()
-        self.client = ApitallyClient(client_id=client_id, env=env)
+        self.client = ApitallyClient(client_id=client_id, env=env, request_logging_config=request_logging_config)
         self.client.start_sync_loop()
         self.delayed_set_startup_data(app_version, openapi_url)
 
+        self.capture_request_body = (
+            self.client.request_logger.config.enabled and self.client.request_logger.config.log_request_body
+        )
+        self.capture_response_body = (
+            self.client.request_logger.config.enabled and self.client.request_logger.config.log_response_body
+        )
+
     def delayed_set_startup_data(self, app_version: Optional[str] = None, openapi_url: Optional[str] = None) -> None:
         # Short delay to allow app routes to be registered first
         timer = Timer(
@@ -56,8 +68,9 @@ class ApitallyMiddleware:
         self.client.set_startup_data(data)
 
     def __call__(self, environ: WSGIEnvironment, start_response: StartResponse) -> Iterable[bytes]:
-        status_code = 200
+        timestamp = time.time()
         response_headers = Headers([])
+        status_code = 0
 
         def catching_start_response(status: str, headers: List[Tuple[str, str]], exc_info=None):
             nonlocal status_code, response_headers
@@ -65,14 +78,41 @@ class ApitallyMiddleware:
             response_headers = Headers(headers)
             return start_response(status, headers, exc_info)
 
-        start_time = time.perf_counter()
         with self.app.app_context():
+            request = Request(environ, populate_request=False, shallow=True)
+            request_size = request.content_length
+            request_body = b""
+            if self.capture_request_body:
+                request_body = (
+                    _read_request_body(environ)
+                    if request_size is not None and request_size <= MAX_BODY_SIZE
+                    else BODY_TOO_LARGE
+                )
+
+            start_time = time.perf_counter()
             response = self.wsgi_app(environ, catching_start_response)
+            response_time = time.perf_counter() - start_time
+
+            response_body = b""
+            if self.capture_response_body:
+                response_size = response_headers.get("Content-Length", type=int)
+                if response_size is not None and response_size > MAX_BODY_SIZE:
+                    response_body = BODY_TOO_LARGE
+                else:
+                    for chunk in response:
+                        response_body += chunk
+                        if len(response_body) > MAX_BODY_SIZE:
+                            response_body = BODY_TOO_LARGE
+                            break
+
             self.add_request(
-                environ=environ,
+                timestamp=timestamp,
+                request=request,
+                request_body=request_body,
                 status_code=status_code,
-                response_time=time.perf_counter() - start_time,
+                response_time=response_time,
                 response_headers=response_headers,
+                response_body=response_body,
             )
         return response
 
@@ -87,41 +127,68 @@ class ApitallyMiddleware:
 
     def add_request(
         self,
-        environ: WSGIEnvironment,
+        timestamp: float,
+        request: Request,
+        request_body: bytes,
         status_code: int,
         response_time: float,
         response_headers: Headers,
+        response_body: bytes,
     ) -> None:
-        rule, is_handled_path = self.get_rule(environ)
-        if (is_handled_path or not self.filter_unhandled_paths) and environ["REQUEST_METHOD"] != "OPTIONS":
-            consumer = self.get_consumer()
-            consumer_identifier = consumer.identifier if consumer else None
-            self.client.consumer_registry.add_or_update_consumer(consumer)
+        path = self.get_path(request.environ)
+        response_size = response_headers.get("Content-Length", type=int)
+
+        consumer = self.get_consumer()
+        consumer_identifier = consumer.identifier if consumer else None
+        self.client.consumer_registry.add_or_update_consumer(consumer)
+
+        if path is not None and request.method != "OPTIONS":
             self.client.request_counter.add_request(
                 consumer=consumer_identifier,
-                method=environ["REQUEST_METHOD"],
-                path=rule,
+                method=request.method,
+                path=path,
                 status_code=status_code,
                 response_time=response_time,
-                request_size=environ.get("CONTENT_LENGTH"),
-                response_size=response_headers.get("Content-Length", type=int),
+                request_size=request.content_length,
+                response_size=response_size,
             )
             if status_code == 500 and "unhandled_exception" in g:
                 self.client.server_error_counter.add_server_error(
                     consumer=consumer_identifier,
-                    method=environ["REQUEST_METHOD"],
-                    path=rule,
+                    method=request.method,
+                    path=path,
                     exception=g.unhandled_exception,
                 )
 
-    def get_rule(self, environ: WSGIEnvironment) -> Tuple[str, bool]:
+        if self.client.request_logger.enabled:
+            self.client.request_logger.log_request(
+                request={
+                    "timestamp": timestamp,
+                    "method": request.method,
+                    "path": path,
+                    "url": request.url,
+                    "headers": list(request.headers.items()),
+                    "size": request.content_length,
+                    "consumer": consumer_identifier,
+                    "body": request_body,
+                },
+                response={
+                    "status_code": status_code,
+                    "response_time": response_time,
+                    "headers": list(response_headers.items()),
+                    "size": response_size,
+                    "body": response_body,
+                },
+            )
+
+    def get_path(self, environ: WSGIEnvironment) -> Optional[str]:
         url_adapter = self.app.url_map.bind_to_environ(environ)
         try:
             endpoint, _ = url_adapter.match()
             rule = self.app.url_map._rules_by_endpoint[endpoint][0]
-            return rule.rule, True
+            return rule.rule
         except NotFound:
-            return environ["PATH_INFO"], False
+            return None
 
     def get_consumer(self) -> Optional[ApitallyConsumer]:
         if "apitally_consumer" in g and g.apitally_consumer:
@@ -166,3 +233,10 @@ def _get_openapi(app: WSGIApplication, openapi_url: str) -> Optional[str]:
     if response.status_code != 200:
         return None
     return response.get_data(as_text=True)
+
+
+def _read_request_body(environ: WSGIEnvironment) -> bytes:
+    length = int(environ.get("CONTENT_LENGTH", "0"))
+    body = environ["wsgi.input"].read(length)
+    environ["wsgi.input"] = BytesIO(body)
+    return body