fetchnode 0.4.0__py3-none-any.whl

fetchnode/celery.py

@@ -0,0 +1,138 @@
+"""Optional Celery signal instrumentation with no Celery hard dependency."""
+
+from __future__ import annotations
+
+import threading
+import time
+import uuid
+
+from fetchnode.api import capture_exception
+from fetchnode.capture import track_job_execution
+
+
+_started = {}
+_finished = {}
+_lock = threading.Lock()
+_installed = False
+
+
+def _request_details(task, task_id: str) -> dict:
+    request = getattr(task, "request", None)
+    headers = getattr(request, "headers", None) or {}
+    delivery = getattr(request, "delivery_info", None) or {}
+    sent_at = headers.get("fetchnode_sent_at")
+    try:
+        queue_latency_ms = max(0, (time.time() - float(sent_at)) * 1000) if sent_at else 0
+    except (TypeError, ValueError):
+        queue_latency_ms = 0
+    return {
+        "task_id": task_id,
+        "task_name": getattr(task, "name", "") or type(task).__name__,
+        "queue": str(delivery.get("routing_key") or ""),
+        "retry_count": int(getattr(request, "retries", 0) or 0),
+        "queue_latency_ms": queue_latency_ms,
+        "trace_id": str(headers.get("fetchnode_trace_id") or uuid.uuid4().hex),
+        "parent_trace_id": str(headers.get("fetchnode_parent_trace_id") or ""),
+    }
+
+
+def _before_publish(headers=None, **kwargs):
+    if headers is None:
+        return
+    headers.setdefault("fetchnode_sent_at", time.time())
+    headers.setdefault("fetchnode_trace_id", uuid.uuid4().hex)
+
+
+def _task_prerun(task_id=None, task=None, **kwargs):
+    details = _request_details(task, str(task_id or ""))
+    with _lock:
+        identifier = str(task_id or "")
+        _started[identifier] = (time.monotonic(), details)
+        _finished.pop(identifier, None)
+
+
+def _pop_details(task_id, task):
+    with _lock:
+        stored = _started.pop(str(task_id or ""), None)
+    if stored:
+        return stored
+    return time.monotonic(), _request_details(task, str(task_id or ""))
+
+
+def _mark_finished(task_id) -> None:
+    now = time.monotonic()
+    identifier = str(task_id or "")
+    with _lock:
+        for stale_id, finished_at in list(_finished.items()):
+            if now - finished_at > 3600:
+                _finished.pop(stale_id, None)
+        _finished[identifier] = now
+
+
+def _consume_finished(task_id) -> bool:
+    with _lock:
+        return _finished.pop(str(task_id or ""), None) is not None
+
+
+def _task_postrun(task_id=None, task=None, state=None, **kwargs):
+    if _consume_finished(task_id):
+        return
+    started, details = _pop_details(task_id, task)
+    normalized_state = str(state or "").upper()
+    status = "succeeded" if normalized_state == "SUCCESS" else "retrying" if normalized_state == "RETRY" else "failed"
+    track_job_execution(
+        **details,
+        status=status,
+        duration_ms=(time.monotonic() - started) * 1000,
+        runtime="celery",
+    )
+
+
+def _task_failure(task_id=None, exception=None, sender=None, **kwargs):
+    task = sender
+    started, details = _pop_details(task_id, task)
+    if exception is not None:
+        capture_exception(
+            exception,
+            tags={"runtime": "celery", "task_name": details["task_name"], "task_id": details["task_id"]},
+            logger_name=f"celery.{details['task_name']}",
+            mechanism="celery.signal",
+            handled=False,
+        )
+    track_job_execution(
+        **details,
+        status="failed",
+        duration_ms=(time.monotonic() - started) * 1000,
+        runtime="celery",
+    )
+    _mark_finished(task_id)
+
+
+def _task_retry(request=None, reason=None, sender=None, **kwargs):
+    task_id = str(getattr(request, "id", "") or "")
+    started, details = _pop_details(task_id, sender)
+    track_job_execution(
+        **details,
+        status="retrying",
+        duration_ms=(time.monotonic() - started) * 1000,
+        runtime="celery",
+        properties={"reason": str(reason or "")[:500]},
+    )
+    _mark_finished(task_id)
+
+
+def install_celery_instrumentation() -> bool:
+    global _installed
+    if _installed:
+        return True
+    try:
+        from celery.signals import before_task_publish, task_failure, task_postrun, task_prerun, task_retry
+    except ImportError:
+        return False
+    before_task_publish.connect(_before_publish, weak=False, dispatch_uid="fetchnode.before_publish")
+    task_prerun.connect(_task_prerun, weak=False, dispatch_uid="fetchnode.task_prerun")
+    task_postrun.connect(_task_postrun, weak=False, dispatch_uid="fetchnode.task_postrun")
+    task_failure.connect(_task_failure, weak=False, dispatch_uid="fetchnode.task_failure")
+    task_retry.connect(_task_retry, weak=False, dispatch_uid="fetchnode.task_retry")
+    _installed = True
+    return True

fetchnode/conf.py

@@ -0,0 +1,127 @@
+import os
+from django.conf import settings
+
+
+DEFAULTS = {
+    "ENABLED": False,
+    "API_URL": "https://monitor.fetchnode.com/api/events/ingest/",
+    "API_KEY": "",
+    "ENVIRONMENT": "development",
+    "RELEASE": "",
+    "TIMEOUT": 2,
+    "MAX_RETRIES": 3,
+    "BATCH_SIZE": 10,
+    "COMPRESS_PAYLOADS": True,
+    "COMPRESSION_THRESHOLD_BYTES": 1024,
+    "DISK_BUFFER_PATH": "",
+    "MAX_BUFFERED_EVENTS": 1000,
+    "SEND_CLIENT_REPORTS": True,
+    "CLIENT_REPORT_INTERVAL_SECONDS": 60,
+    "TRANSACTION_SAMPLE_RATE": 1.0,
+    "JOB_SAMPLE_RATE": 1.0,
+    "KEEP_TRANSACTION_OUTLIER_MS": 1000,
+    "CAPTURE_TRANSACTIONS": True,
+    "CAPTURE_JOBS": True,
+    "INSTRUMENT_CELERY": True,
+    "INSTRUMENT_LOGGING": True,
+    "INSTRUMENT_EXCEPTHOOK": True,
+    "IGNORE_PATH_PREFIXES": [
+        "/api/events/ingest/", "/widget/", "/static/", "/admin/jsi18n/", "/health/", "/ready/",
+    ],
+    "CAPTURE_HEADERS": False,
+    "CAPTURE_BODY": False,
+    "CAPTURE_IP_ADDRESS": False,
+    "CAPTURE_QUERY_STRING": False,
+    "CAPTURE_QUERY_TEXT": False,
+    "CAPTURE_QUERY_SHAPE": True,
+    "QUERY_SLOW_THRESHOLD_MS": 100,
+    "N_PLUS_ONE_THRESHOLD": 5,
+    "MAX_QUERY_OBSERVATIONS": 50,
+    "TRACE_RESPONSE_HEADER": True,
+    "CAPTURE_USER_ID": True,
+    "CAPTURE_USER_EMAIL": False,
+    "CAPTURE_USERNAME": False,
+    "MAX_BODY_LENGTH": 4000,
+    "SENSITIVE_KEYS": [
+        "password",
+        "passwd",
+        "secret",
+        "token",
+        "api_key",
+        "access_token",
+        "refresh_token",
+        "authorization",
+        "cookie",
+        "csrfmiddlewaretoken",
+    ],
+    "SENSITIVE_HEADERS": ["authorization", "cookie", "x-api-key"],
+}
+
+
+def get_client_settings() -> dict:
+    configured = getattr(settings, "FETCHNODE", None)
+    if configured is None:
+        configured = getattr(settings, "FETCHNODE_CLIENT", {})
+    if not isinstance(configured, dict):
+        configured = {}
+
+    merged = DEFAULTS.copy()
+    merged.update(configured)
+
+    # API key resolution: API_KEY, KEY, PROJECT_KEY, or env
+    api_key = (
+        configured.get("API_KEY")
+        or configured.get("KEY")
+        or configured.get("PROJECT_KEY")
+        or os.environ.get("FETCHNODE_API_KEY", "")
+        or os.environ.get("FETCHNODE_KEY", "")
+    )
+    merged["API_KEY"] = str(api_key or "").strip()
+
+    # URL resolution: API_URL, ENDPOINT, DSN, HOST, or env
+    api_url = (
+        configured.get("API_URL")
+        or configured.get("ENDPOINT")
+        or configured.get("DSN")
+        or configured.get("HOST")
+        or os.environ.get("FETCHNODE_API_URL", "")
+        or os.environ.get("FETCHNODE_ENDPOINT", "")
+        or DEFAULTS["API_URL"]
+    )
+    api_url = str(api_url or "").strip()
+    if api_url and not api_url.rstrip("/").endswith(("/api/events/ingest", "/api/events", "/events")):
+        api_url = api_url.rstrip("/") + "/api/events/ingest/"
+    merged["API_URL"] = api_url
+
+    # Environment resolution
+    env = (
+        configured.get("ENVIRONMENT")
+        or os.environ.get("FETCHNODE_ENVIRONMENT")
+        or getattr(settings, "DJANGO_ENV", "")
+        or "development"
+    )
+    merged["ENVIRONMENT"] = str(env or "").strip()
+
+    # Release resolution
+    release = (
+        configured.get("RELEASE")
+        or os.environ.get("FETCHNODE_RELEASE")
+        or getattr(settings, "GIT_COMMIT", "")
+        or ""
+    )
+    merged["RELEASE"] = str(release or "").strip()
+
+    # Enabled: if specified explicitly, respect it.
+    # Otherwise auto-enable if an API key is present!
+    if "ENABLED" in configured:
+        merged["ENABLED"] = bool(configured["ENABLED"])
+    elif "FETCHNODE_ENABLED" in os.environ:
+        merged["ENABLED"] = os.environ["FETCHNODE_ENABLED"].lower() in {"1", "true", "yes"}
+    else:
+        merged["ENABLED"] = bool(merged["API_KEY"])
+
+    # Security filters are additive: a project can protect extra business
+    # fields without accidentally unprotecting cookies or authorization data.
+    for field in ("SENSITIVE_KEYS", "SENSITIVE_HEADERS"):
+        merged[field] = list(dict.fromkeys([*DEFAULTS[field], *configured.get(field, [])]))
+    return merged

fetchnode/demo.py

@@ -0,0 +1,3 @@
+def boom():
+    raise ValueError("Boom from demo helper")
+

fetchnode/excepthook.py

@@ -0,0 +1,83 @@
+"""Global unhandled exception hook for scripts, cron jobs, and management commands."""
+
+from __future__ import annotations
+
+import sys
+import threading
+from typing import Any, Callable
+
+from fetchnode.api import capture_exception
+from fetchnode.sender import flush
+
+_original_excepthook: Callable[..., Any] | None = None
+_installed: bool = False
+_lock = threading.Lock()
+
+
+def fetchnode_excepthook(exc_type, exc_value, exc_traceback) -> None:
+    """Capture unhandled exceptions, flush the sender queue, and chain to the original hook."""
+    try:
+        is_ignored = (
+            (isinstance(exc_type, type) and issubclass(exc_type, (KeyboardInterrupt, SystemExit)))
+            or isinstance(exc_value, (KeyboardInterrupt, SystemExit))
+        )
+        if not is_ignored:
+            exc = exc_value
+            if exc is None and isinstance(exc_type, type) and issubclass(exc_type, BaseException):
+                try:
+                    exc = exc_type()
+                except Exception:
+                    try:
+                        exc = BaseException.__new__(exc_type)
+                    except Exception:
+                        exc = None
+            if exc is not None and isinstance(exc, BaseException):
+                if exc_traceback is not None and getattr(exc, "__traceback__", None) is not exc_traceback:
+                    try:
+                        exc = exc.with_traceback(exc_traceback)
+                    except Exception:
+                        pass
+                capture_exception(exc, mechanism="excepthook", handled=False)
+                flush(timeout=2.0)
+    except Exception:
+        pass
+    finally:
+        hook = _original_excepthook
+        if hook is None or hook is fetchnode_excepthook:
+            hook = getattr(sys, "__excepthook__", None)
+        if hook is not None and hook is not fetchnode_excepthook:
+            hook(exc_type, exc_value, exc_traceback)
+
+
+def install_excepthook_instrumentation() -> bool:
+    """Install sys.excepthook to capture unhandled process crashes."""
+    global _installed, _original_excepthook
+    with _lock:
+        if sys.excepthook is fetchnode_excepthook:
+            _installed = True
+            if _original_excepthook is None or _original_excepthook is fetchnode_excepthook:
+                _original_excepthook = getattr(sys, "__excepthook__", None)
+            return True
+        _original_excepthook = sys.excepthook
+        sys.excepthook = fetchnode_excepthook
+        _installed = True
+        return True
+
+
+def uninstall_excepthook_instrumentation() -> bool:
+    """Restore sys.excepthook to the previous handler."""
+    global _installed, _original_excepthook
+    with _lock:
+        if not _installed and sys.excepthook is not fetchnode_excepthook:
+            return False
+        if sys.excepthook is fetchnode_excepthook:
+            target = _original_excepthook or getattr(sys, "__excepthook__", None)
+            if target is fetchnode_excepthook:
+                target = getattr(sys, "__excepthook__", None)
+            sys.excepthook = target
+        _original_excepthook = None
+        _installed = False
+        return True
+
+
+restore_excepthook = uninstall_excepthook_instrumentation

fetchnode/handlers.py

@@ -0,0 +1,76 @@
+import logging
+
+from fetchnode.api import build_exception_payload, capture_message
+from fetchnode.excepthook import (
+    fetchnode_excepthook,
+    install_excepthook_instrumentation,
+    restore_excepthook,
+    uninstall_excepthook_instrumentation,
+)
+from fetchnode.sender import send_payload
+from fetchnode.scope import get_request
+
+
+_CAPTURED_RECORD_ATTRIBUTE = "_fetchnode_logging_captured"
+
+
+class FetchNodeHandler(logging.Handler):
+    def emit(self, record):
+        try:
+            self._emit(record)
+        except Exception:
+            self.handleError(record)
+
+    def _emit(self, record):
+        if (
+            record.levelno < logging.ERROR
+            or record.name == "fetchnode"
+            or record.name.startswith("fetchnode.")
+            or getattr(record, _CAPTURED_RECORD_ATTRIBUTE, False)
+        ):
+            return
+        setattr(record, _CAPTURED_RECORD_ATTRIBUTE, True)
+        request = get_request()
+        if not record.exc_info:
+            kwargs = {
+                "level": record.levelname.lower(),
+                "logger_name": record.name,
+                "tags": {"logger_handler": "logging"},
+            }
+            if request is not None:
+                kwargs["request"] = request
+            capture_message(record.getMessage(), **kwargs)
+            return
+        exc_value = record.exc_info[1]
+        if exc_value is not None:
+            kwargs = {
+                "logger_name": record.name,
+                "tags": {"logger_handler": "logging"},
+                "mechanism": "logging",
+                "handled": True,
+            }
+            if request is not None:
+                kwargs["request"] = request
+            payload = build_exception_payload(exc_value, **kwargs)
+            payload["level"] = record.levelname.lower()
+            send_payload(payload)
+
+
+def install_logging_instrumentation() -> FetchNodeHandler:
+    """Attach one ERROR+ handler to root and non-propagating application loggers."""
+    root_logger = logging.getLogger()
+    handler = next(
+        (handler for handler in root_logger.handlers if isinstance(handler, FetchNodeHandler)),
+        None,
+    )
+    if handler is None:
+        handler = FetchNodeHandler(level=logging.ERROR)
+        root_logger.addHandler(handler)
+    for application_logger in logging.root.manager.loggerDict.values():
+        if (
+            isinstance(application_logger, logging.Logger)
+            and not application_logger.propagate
+            and handler not in application_logger.handlers
+        ):
+            application_logger.addHandler(handler)
+    return handler

fetchnode/middleware.py

@@ -0,0 +1,188 @@
+import time
+import uuid
+import re
+
+from django.db import connection
+
+from fetchnode.api import capture_exception
+from fetchnode.conf import get_client_settings
+from fetchnode.query import record_query_observation, serialize_query_observations
+from fetchnode.scope import add_breadcrumb, clear_scope, set_request
+from fetchnode.sender import send_payload
+from fetchnode.utils import utc_now_iso
+
+
+_TRACEPARENT = re.compile(r"^[\da-f]{2}-([\da-f]{32})-[\da-f]{16}-[\da-f]{2}$", re.IGNORECASE)
+_SAFE_CORRELATION_ID = re.compile(r"^[\w.-]{1,64}$")
+
+
+def _request_correlation(request) -> tuple[str, str]:
+    traceparent = request.headers.get("Traceparent", "")
+    match = _TRACEPARENT.fullmatch(traceparent.strip())
+    supplied_trace = request.headers.get("X-FetchNode-Trace-ID", "").strip().lower()
+    trace_id = match.group(1).lower() if match else supplied_trace
+    if not re.fullmatch(r"[\da-f]{32}", trace_id):
+        trace_id = uuid.uuid4().hex
+    supplied_request_id = request.headers.get("X-Request-ID", "").strip()
+    request_id = supplied_request_id if _SAFE_CORRELATION_ID.fullmatch(supplied_request_id) else uuid.uuid4().hex
+    return trace_id, request_id
+
+
+def _safe_header_id(request, name: str) -> str:
+    value = request.headers.get(name, "").strip()
+    return value if _SAFE_CORRELATION_ID.fullmatch(value) else ""
+
+
+class ErrorCaptureMiddleware:
+    """Capture unhandled Django exceptions with request and database context."""
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+        try:
+            from django.conf import settings
+            from fetchnode.api import track_event
+            setattr(settings, "fetchnode_track", track_event)
+            setattr(settings, "FETCHNODE_TRACK", track_event)
+        except Exception:
+            pass
+
+    def __call__(self, request):
+        config = get_client_settings()
+        if not config["ENABLED"]:
+            return self.get_response(request)
+        clear_scope()
+        set_request(request)
+        try:
+            return self._handle_request(request, config)
+        finally:
+            clear_scope()
+
+    def _handle_request(self, request, config):
+        started_request = time.monotonic()
+        trace_id, request_id = _request_correlation(request)
+        request._fetchnode_trace_id = trace_id
+        request._fetchnode_request_id = request_id
+        request._fetchnode_transaction_event_id = uuid.uuid4().hex
+        add_breadcrumb(
+            f"{request.method} {request.path}",
+            category="http.request",
+            data={"method": request.method, "path": request.path},
+        )
+        queries = []
+        query_stats = {"count": 0, "duration_ms": 0.0, "observations": {}}
+        request._fetchnode_queries = queries
+        request._fetchnode_query_stats = query_stats
+
+        def query_log(execute, sql, params, many, context):
+            started = time.monotonic()
+            try:
+                return execute(sql, params, many, context)
+            finally:
+                duration_ms = round((time.monotonic() - started) * 1000, 2)
+                query_stats["count"] += 1
+                query_stats["duration_ms"] += duration_ms
+                if len(queries) < 50:
+                    query = {"duration_ms": duration_ms}
+                    if config["CAPTURE_QUERY_TEXT"]:
+                        query["sql"] = str(sql)[:20_000]
+                    queries.append(query)
+                record_query_observation(
+                    query_stats["observations"],
+                    sql=str(sql),
+                    duration_ms=duration_ms,
+                    slow_threshold_ms=float(config["QUERY_SLOW_THRESHOLD_MS"]),
+                    n_plus_one_threshold=int(config["N_PLUS_ONE_THRESHOLD"]),
+                    capture_shape=bool(config["CAPTURE_QUERY_SHAPE"]),
+                    maximum=int(config["MAX_QUERY_OBSERVATIONS"]),
+                )
+
+        try:
+            with connection.execute_wrapper(query_log):
+                response = self.get_response(request)
+        except Exception as exc:
+            if not getattr(request, "_fetchnode_exception_captured", False):
+                request._fetchnode_exception_captured = True
+                self.process_exception(request, exc)
+            self._send_transaction(request, 500, started_request, query_stats, config)
+            raise
+        if config["CAPTURE_TRANSACTIONS"] and not any(
+            request.path.startswith(prefix) for prefix in config["IGNORE_PATH_PREFIXES"]
+        ):
+            self._send_transaction(request, response.status_code, started_request, query_stats, config)
+        if config["TRACE_RESPONSE_HEADER"]:
+            response.setdefault("X-FetchNode-Trace-ID", trace_id)
+            response.setdefault("X-Request-ID", request_id)
+        return response
+
+    def _send_transaction(self, request, status_code, started_request, query_stats, config=None):
+        config = config or get_client_settings()
+        if not config["CAPTURE_TRANSACTIONS"] or any(
+            request.path.startswith(prefix) for prefix in config["IGNORE_PATH_PREFIXES"]
+        ):
+            return
+        resolver_match = getattr(request, "resolver_match", None)
+        auth_user = getattr(request, "user", None)
+        user_identifier = ""
+        if config["CAPTURE_USER_ID"] and auth_user is not None and getattr(auth_user, "is_authenticated", False):
+            user_identifier = str(getattr(auth_user, "pk", ""))[:255]
+        send_payload({
+            "event_id": request._fetchnode_transaction_event_id,
+            "type": "transaction",
+            "timestamp": utc_now_iso(),
+            "method": request.method,
+            "path": request.path,
+            "route": getattr(resolver_match, "route", "") or "",
+            "view_name": getattr(resolver_match, "view_name", "") or "",
+            "status_code": status_code,
+            "duration_ms": round((time.monotonic() - started_request) * 1000, 2),
+            "db_queries_count": query_stats["count"],
+            "db_time_ms": round(query_stats["duration_ms"], 2),
+            "query_observations": serialize_query_observations(query_stats["observations"]),
+            "environment": config["ENVIRONMENT"],
+            "release": config["RELEASE"],
+            "trace_id": request._fetchnode_trace_id,
+            "request_id": request._fetchnode_request_id,
+            "user_identifier": user_identifier,
+            "visitor_id": (
+                _safe_header_id(request, "X-FetchNode-Visitor-ID")
+                or getattr(request, "COOKIES", {}).get("_fn_vid", "")
+            ),
+            "session_id": (
+                _safe_header_id(request, "X-FetchNode-Session-ID")
+                or getattr(request, "COOKIES", {}).get("_fn_sid", "")
+            ),
+        })
+
+    def process_exception(self, request, exception):
+        config = get_client_settings()
+        if not config["ENABLED"] or any(request.path.startswith(prefix) for prefix in config["IGNORE_PATH_PREFIXES"]):
+            clear_scope()
+            return None
+        queries = getattr(request, "_fetchnode_queries", [])
+        query_stats = getattr(
+            request,
+            "_fetchnode_query_stats",
+            {"count": len(queries), "duration_ms": sum(query["duration_ms"] for query in queries)},
+        )
+        capture_exception(
+            exception,
+            request=request,
+            mechanism="django.middleware",
+            handled=False,
+            queries=queries,
+            contexts={
+                "database": {
+                    "query_count": query_stats["count"],
+                    "query_time_ms": round(query_stats["duration_ms"], 2),
+                    "truncated": query_stats["count"] > len(queries),
+                }
+            },
+        )
+        request._fetchnode_exception_captured = True
+        clear_scope()
+        return None
+
+
+# Primary and recommended middleware class name:
+FetchNodeMiddleware = ErrorCaptureMiddleware
+