varanus 0.1.0__py3-none-any.whl

varanus/__init__.py

@@ -0,0 +1,6 @@
+from .version import __version__, __version_info__
+
+__all__ = [
+    "__version__",
+    "__version_info__",
+]

varanus/client/__init__.py

@@ -0,0 +1,7 @@
+from .client import client
+
+setup = client.setup
+context = client.context
+log = client.log
+metric = client.metric
+timer = client.timer

varanus/client/apps.py

@@ -0,0 +1,8 @@
+from django.apps import AppConfig
+
+
+class VaranusClient(AppConfig):
+    name = "varanus.client"
+
+    def ready(self):
+        pass

varanus/client/client.py

@@ -0,0 +1,231 @@
+import logging
+import os
+import platform
+import warnings
+from importlib.metadata import distributions
+from typing import Iterable
+from urllib.parse import urlsplit
+
+import varanus
+from varanus.events import Context, NodeInfo
+
+from ..utils import import_string
+from .context import VaranusContext, current_context
+from .loggers import QueryLogger
+from .transport.base import BaseTransport
+
+
+def install_query_logger(logger):
+    def handler(sender, **kwargs):
+        if logger not in kwargs["connection"].execute_wrappers:
+            kwargs["connection"].execute_wrappers.append(logger)
+
+    return handler
+
+
+def resolve_include_exclude(
+    items: Iterable[str],
+    include: Iterable[str] | bool,
+    exclude: Iterable[str] | None,
+) -> set[str]:
+    if not include:
+        return set()
+
+    if include is True:
+        resolved = set(items)
+    else:
+        resolved = set(items).intersection(include)
+
+    if exclude is not None:
+        resolved.difference_update(exclude)
+
+    return resolved
+
+
+class VaranusClient:
+    # Things we don't want to send, at least by default.
+    sensitive_headers = set(["authorization", "cookie", "proxy-authorization"])
+    sensitive_settings = set(["SECRET_KEY"])
+    sensitive_env = set(["PGPASSWORD"])
+
+    scheme_transports = {
+        "http": "varanus.client.transport.http.ThreadedHttpTransport",
+        "https": "varanus.client.transport.http.ThreadedHttpTransport",
+        "db": "varanus.client.transport.database.ModelTransport",
+    }
+
+    configured = False
+
+    def setup(
+        self,
+        dsn: str,
+        environment: str,
+        node: str | None = None,
+        transport_class: str | type[BaseTransport] | None = None,
+        request_attr: str = "varanus",
+        logger_name: str = "varanus.request",
+        log_warnings: bool = True,
+        tags: dict | None = None,
+        include_headers: Iterable[str] | bool = False,
+        exclude_headers: Iterable[str] | None = None,
+        include_settings: Iterable[str] | bool = True,
+        exclude_settings: Iterable[str] | None = None,
+        include_default_settings: bool = False,
+        filter_settings: bool = True,
+        include_env: Iterable[str] | bool = False,
+        exclude_env: Iterable[str] | None = None,
+        log_queries: bool | int = False,
+        log_query_params: bool = False,
+        log_query_stack: bool = False,
+        query_metrics: bool | str = False,
+        send_all: bool = False,
+        install: list | None = None,
+    ):
+        url = urlsplit(dsn)
+        self.environment = environment
+        self.node = node or platform.node()
+        if transport_class is None:
+            if url.scheme not in self.scheme_transports:
+                raise ValueError(f"No transport class found for `{url.scheme}`")
+            resolved_class = import_string(self.scheme_transports[url.scheme])
+        elif isinstance(transport_class, str):
+            resolved_class = import_string(transport_class)
+        else:
+            resolved_class = transport_class
+        if not issubclass(resolved_class, BaseTransport):
+            raise ValueError(
+                f"Transport class `{transport_class}` must be a subclass of"
+                "BaseTransport."
+            )
+        self.transport = resolved_class(url, self.environment, self.node)
+        self.request_attr = request_attr
+        self.logger_name = logger_name
+        self.tags = tags or {}
+        self.include_headers = include_headers
+        self.exclude_headers = (
+            self.sensitive_headers if exclude_headers is None else set(exclude_headers)
+        )
+        self.include_settings = include_settings
+        self.exclude_settings = (
+            self.sensitive_settings
+            if exclude_settings is None
+            else set(exclude_settings)
+        )
+        self.include_default_settings = include_default_settings
+        self.filter_settings = filter_settings
+        self.include_env = include_env
+        self.exclude_env = (
+            self.sensitive_env if exclude_env is None else set(exclude_env)
+        )
+        if log_queries or query_metrics:
+            try:
+                # The logger is installed as early as possible, and for all connections.
+                from django.db.backends.signals import connection_created
+
+                # Create a single QueryLogger to be used by all connections.
+                self.query_logger = QueryLogger(
+                    log_queries,
+                    log_query_params,
+                    log_query_stack,
+                    query_metrics,
+                )
+                # Install it in each new connection (if it's not already installed).
+                connection_created.connect(
+                    install_query_logger(self.query_logger),
+                    weak=False,
+                )
+            except ImportError:
+                pass
+        self.send_all = send_all
+        if log_warnings:
+            warnings.simplefilter("default")
+            logging.captureWarnings(True)
+        self.configured = True
+        if install:
+            if "django.contrib.auth.middleware.AuthenticationMiddleware" in install:
+                idx = install.index(
+                    "django.contrib.auth.middleware.AuthenticationMiddleware"
+                )
+                install.insert(idx + 1, "varanus.client.middleware.VaranusMiddleware")
+            elif "django.middleware.common.CommonMiddleware" in install:
+                idx = install.index("django.middleware.common.CommonMiddleware")
+                install.insert(idx + 1, "varanus.client.middleware.VaranusMiddleware")
+            else:
+                install.append("varanus.client.middleware.VaranusMiddleware")
+        return self
+
+    def send(self, *events: Context):
+        for e in events:
+            self.transport.send(e)
+
+    def ping(self):
+        from django import get_version
+        from django.conf import settings
+
+        if self.filter_settings:
+            from django.views.debug import SafeExceptionReporterFilter
+
+            settings_dict = {
+                k: repr(v)
+                for k, v in SafeExceptionReporterFilter().get_safe_settings().items()
+            }
+        else:
+            settings_dict = {
+                s: repr(getattr(settings, s)) for s in dir(settings) if s.isupper()
+            }
+
+        include_settings = resolve_include_exclude(
+            settings_dict.keys(),
+            self.include_settings,
+            self.exclude_settings,
+        )
+        include_env = resolve_include_exclude(
+            os.environ.keys(),
+            self.include_env,
+            self.exclude_env,
+        )
+
+        self.transport.ping(
+            NodeInfo(
+                name=self.node,
+                version=varanus.__version__,
+                platform=platform.platform(),
+                language=platform.python_implementation(),
+                language_version=platform.python_version(),
+                framework="Django",
+                framework_version=get_version(),
+                packages={d.name: d.version for d in distributions()},
+                settings={
+                    s: settings_dict[s]
+                    for s in include_settings
+                    if self.include_default_settings or settings.is_overridden(s)
+                },
+                environment={e: os.environ[e] for e in include_env},
+            )
+        )
+
+    def log(self, level: int, message: str, *args, **kwargs):
+        if ctx := current_context.get():
+            kwargs.setdefault("stacklevel", 2)
+            ctx.log(level, message, *args, **kwargs)
+
+    def raw_exception(self, exception, tags: dict | None = None):
+        if ctx := current_context.get():
+            ctx.raw_exception(exception, tags=tags)
+
+    def metric(self, name: str, value: float = 0.0, tags: dict | None = None):
+        if ctx := current_context.get():
+            ctx.metric(name, value, tags=tags)
+
+    def timer(self, name: str, tags: dict | None = None):
+        if ctx := current_context.get():
+            return ctx.timer(name, tags=tags)
+
+    def context(self, name: str, tags: dict | None = None):
+        if ctx := current_context.get():
+            return ctx.context(name, tags)
+        else:
+            return VaranusContext(self, name, tags or self.tags)
+
+
+client = VaranusClient()

varanus/client/context.py

@@ -0,0 +1,156 @@
+import contextlib
+import inspect
+import logging
+import sys
+import time
+from contextvars import ContextVar, Token
+from datetime import timedelta
+from typing import TYPE_CHECKING
+
+from ..events import Context, Error, Log, Metric, Request, now
+
+if TYPE_CHECKING:
+    from .client import VaranusClient
+
+ONE_MS = timedelta(milliseconds=1)
+
+
+class VaranusContext:
+    """
+    Shared public interface for how clients add logs/errors/metrics/etc. to Varanus.
+    """
+
+    request: Request | None
+
+    def __init__(self, client: "VaranusClient", name, tags: dict | None = None):
+        self.client = client
+        self.name = name
+        self.logs = []
+        self.errors = []
+        self.metrics: dict[str, Metric] = {}
+        self.queries = []
+        self.subcontexts = []
+        self.tags = tags or {}
+        self.request = None
+
+    def build(self):
+        return Context(
+            timestamp=self.started,
+            tags=self.tags,
+            name=self.name,
+            elapsed_ms=self.elapsed_ms,
+            request=self.request,
+            logs=self.logs,
+            errors=self.errors,
+            metrics=list(self.metrics.values()),
+            queries=self.queries,
+            subcontexts=[ctx.build() for ctx in self.subcontexts],
+        )
+
+    def should_send(self):
+        if self.client.send_all:
+            return True
+        if self.logs or self.errors or self.metrics or self.queries:
+            return True
+        for ctx in self.subcontexts:
+            if ctx.should_send():
+                return True
+        return False
+
+    def __enter__(self):
+        self.token = current_context.set(self)
+        self.started = now()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self.raw_exception((exc_type, exc_value, traceback))
+        self.elapsed_ms = (now() - self.started) // ONE_MS
+        current_context.reset(self.token)
+        if self.token.old_value is Token.MISSING and self.should_send():
+            # We're the bottom of the stack, build and send the event.
+            try:
+                self.client.send(self.build())
+            except Exception as e:
+                import traceback
+
+                traceback.print_exception(e)
+
+    def __setitem__(self, name, value):
+        self.tags[name] = value
+
+    def log(
+        self,
+        level: int,
+        message: str,
+        *args,
+        exc_info=None,
+        stacklevel: int = 1,
+        tags: dict | None = None,
+        **kwargs,
+    ):
+        frame = inspect.stack()[stacklevel]
+        self.logs.append(
+            Log(
+                tags=tags or {},
+                message=message % args,
+                name=self.client.logger_name,
+                level=level,
+                file=frame.filename,
+                lineno=frame.lineno,
+                error=Error.from_exception(exc_info),
+            )
+        )
+
+    def debug(self, message: str, *args, **kwargs):
+        kwargs.setdefault("stacklevel", 2)
+        self.log(logging.DEBUG, message, *args, **kwargs)
+
+    def info(self, message: str, *args, **kwargs):
+        kwargs.setdefault("stacklevel", 2)
+        self.log(logging.INFO, message, *args, **kwargs)
+
+    def warning(self, message: str, *args, **kwargs):
+        kwargs.setdefault("stacklevel", 2)
+        self.log(logging.WARNING, message, *args, **kwargs)
+
+    def error(self, message: str, *args, **kwargs):
+        kwargs.setdefault("stacklevel", 2)
+        self.log(logging.ERROR, message, *args, **kwargs)
+
+    def critical(self, message: str, *args, **kwargs):
+        kwargs.setdefault("stacklevel", 2)
+        self.log(logging.CRITICAL, message, *args, **kwargs)
+
+    def exception(self, message: str, *args, **kwargs):
+        kwargs.setdefault("stacklevel", 2)
+        kwargs.setdefault("exc_info", sys.exc_info())
+        self.log(logging.ERROR, message, *args, **kwargs)
+
+    def raw_exception(self, exception, tags: dict | None = None):
+        if err := Error.from_exception(exception, tags=tags):
+            self.errors.append(err)
+
+    def context(self, name: str = "", tags: dict | None = None):
+        ctx = VaranusContext(self.client, name, tags)
+        self.subcontexts.append(ctx)
+        return ctx
+
+    def metric(self, name: str, value: float = 0.0, tags: dict | None = None):
+        if name not in self.metrics:
+            self.metrics[name] = Metric(name=name, tags=tags or {})
+        self.metrics[name].update(value)
+
+    @contextlib.contextmanager
+    def timer(self, name: str, tags: dict | None = None):
+        start = time.monotonic()
+        try:
+            yield self
+        finally:
+            elapsed = time.monotonic() - start
+            self.metric(name, elapsed, tags=tags)
+
+
+current_context: ContextVar[VaranusContext | None] = ContextVar(
+    "current_context",
+    default=None,
+)

varanus/client/loggers.py

@@ -0,0 +1,60 @@
+import logging
+import sys
+
+from ..events import Log, Query, capture_stack, now
+from .context import ONE_MS, current_context
+
+
+class VaranusHandler(logging.Handler):
+    def emit(self, record: logging.LogRecord):
+        if ctx := current_context.get():
+            ctx.logs.append(Log.from_logrecord(record, tags=ctx.tags))
+
+
+class QueryLogger:
+    def __init__(self, threshold, log_params, log_stack, metrics):
+        # TODO: add callback for tagging?
+        if threshold is True:
+            self.threshold = 0
+        elif threshold is False:
+            self.threshold = sys.maxsize
+        else:
+            self.threshold = int(threshold)
+        self.log_params = log_params
+        self.log_stack = log_stack
+        if isinstance(metrics, str):
+            self.metrics_name = metrics
+        else:
+            self.metrics_name = "queries" if metrics else None
+
+    def __call__(self, execute, sql, params, many, context):
+        start = now()
+        success = True
+        try:
+            result = execute(sql, params, many, context)
+        except Exception:
+            success = False
+            raise
+        else:
+            return result
+        finally:
+            if ctx := current_context.get():
+                elapsed_ms = (now() - start) // ONE_MS
+                if self.metrics_name:
+                    ctx.metric(self.metrics_name, elapsed_ms)
+                if elapsed_ms >= self.threshold:
+                    ctx.queries.append(
+                        Query(
+                            timestamp=start,
+                            sql=sql,
+                            params=(
+                                [repr(p) for p in params]
+                                if params and self.log_params
+                                else []
+                            ),
+                            db=context["connection"].alias,
+                            elapsed_ms=elapsed_ms,
+                            success=success,
+                            stack=capture_stack(1) if self.log_stack else [],
+                        )
+                    )

varanus/client/middleware.py

@@ -0,0 +1,84 @@
+import ipaddress
+
+from django.core.exceptions import MiddlewareNotUsed
+from django.http import HttpRequest, HttpResponse
+from django.utils import timezone
+
+from ..events import Request
+from .client import client, resolve_include_exclude
+
+
+def get_ip(request: HttpRequest):
+    ip_address = request.META.get("HTTP_X_FORWARDED_FOR", "").strip()
+    if ip_address:
+        ip_address = ip_address.split(",")[0].strip()
+    if not ip_address:
+        ip_address = request.META.get("REMOTE_ADDR", "127.0.0.1").strip()
+    try:
+        # Validate and normalize the IP address.
+        return str(ipaddress.ip_address(ip_address))
+    except ValueError:
+        return ""
+
+
+def request_headers(request: HttpRequest):
+    headers = {}
+    include_headers = resolve_include_exclude(
+        [name.lower() for name in request.headers],
+        client.include_headers,
+        client.exclude_headers,
+    )
+    for name in include_headers:
+        value = request.headers.get(name)
+        if value is not None:
+            headers[name] = value
+    return headers
+
+
+class VaranusMiddleware:
+    def __init__(self, get_response):
+        if not client.configured:
+            # TODO: warning
+            print("VaranusClient is not configured -- disabling middleware.")
+            raise MiddlewareNotUsed()
+        self.last_ping = None
+        self.get_response = get_response
+
+    def process_exception(self, request, exception):
+        # Any value in using request.varanus instead of current context here?
+        client.raw_exception(exception)
+
+    def __call__(self, request: HttpRequest):
+        if self.last_ping is None:
+            self.last_ping = timezone.now()
+            try:
+                client.ping()
+            except Exception:
+                pass
+
+        with client.context(request.path) as varanus:
+            setattr(request, client.request_attr, varanus)
+            response = self.get_response(request)
+            # TODO: any need for request tags separate from context tags?
+            varanus.request = Request(
+                host=request.get_host(),
+                method=request.method or "",
+                path=request.path,
+                query=request.META.get("QUERY_STRING", ""),
+                status=response.status_code,
+                headers=request_headers(request),
+                size=(
+                    len(response.content)
+                    if isinstance(response, HttpResponse)
+                    else None
+                ),
+                ip=get_ip(request),
+                user=(
+                    request.user.get_username()
+                    if hasattr(request, "user")
+                    and request.user
+                    and request.user.is_authenticated
+                    else None
+                ),
+            )
+        return response

varanus/client/transport/base.py

@@ -0,0 +1,14 @@
+from urllib.parse import SplitResult
+
+from varanus import events
+
+
+class BaseTransport:
+    def __init__(self, url: SplitResult, environment: str, node: str):
+        pass
+
+    def ping(self, info: events.NodeInfo):
+        pass
+
+    def send(self, event: events.Context):
+        pass

varanus/client/transport/database.py

@@ -0,0 +1,57 @@
+import typing
+from urllib.parse import SplitResult
+
+import msgspec
+
+from varanus import events
+
+from .base import BaseTransport
+
+if typing.TYPE_CHECKING:
+    from varanus.server.models import Node, Site
+
+
+class ModelTransport(BaseTransport):
+    site: "Site"
+    node: "Node"
+
+    def __init__(self, url: SplitResult, environment: str, node: str):
+        self.slug = url.netloc
+        self.environment = environment
+        self.node_name = node
+
+    def ensure_site(self):
+        if hasattr(self, "site"):
+            return
+
+        from varanus.server.models import Site
+
+        self.site, created = Site.objects.get_or_create(
+            slug=self.slug,
+            defaults={
+                "name": self.slug,
+                "schema_name": self.slug,
+            },
+        )
+
+    def ping(self, info: events.NodeInfo):
+        from varanus.server.models import Node
+
+        self.ensure_site()
+
+        with self.site.activated():
+            self.node, created, updates = Node.update(
+                info, site=self.site, environment=self.environment
+            )
+
+    def send(self, event: events.Context):
+        from varanus.server.tasks import ingest
+
+        self.ensure_site()
+
+        ingest.enqueue(
+            self.site.pk,
+            self.node_name,
+            self.environment,
+            msgspec.json.encode(event).decode(),
+        )