varanus 0.1.0.dev2__py3-none-any.whl → 0.1.0.dev4__py3-none-any.whl

varanus/client/apps.py

@@ -5,4 +5,4 @@ class VaranusClient(AppConfig):
     name = "varanus.client"
 
     def ready(self):
-        print("VaranusClient.ready")
+        pass

varanus/client/client.py

@@ -1,6 +1,7 @@
+import os
 import platform
 from importlib.metadata import distributions
-from typing import Union
+from typing import Iterable
 from urllib.parse import urlsplit
 
 from varanus.events import Context, NodeInfo
@@ -19,64 +20,96 @@ def install_query_logger(logger):
     return handler
 
 
+def resolve_include_exclude(
+    items: Iterable[str],
+    include: Iterable[str] | bool,
+    exclude: Iterable[str] | None,
+) -> set[str]:
+    if not include:
+        return set()
+
+    if include is True:
+        resolved = set(items)
+    else:
+        resolved = set(items).intersection(include)
+
+    if exclude is not None:
+        resolved.difference_update(exclude)
+
+    return resolved
+
+
 class VaranusClient:
-    environment: str | None
-    transport: BaseTransport
-
-    request_attr: str
-    logger_name: str
-    tags: dict
-
-    include_headers: Union[list, bool, None]
-    exclude_headers: list | None
-    sensitive_headers = set(
-        [
-            "authorization",
-            "cookie",
-            "proxy-authorization",
-        ]
-    )
+    sensitive_headers = set(["authorization", "cookie", "proxy-authorization"])
+    sensitive_settings = set(["DATABASES"])
+    sensitive_env = set(["PGPASSWORD"])
 
     scheme_transports = {
-        "test": "varanus.client.transport.test.TestTransport",
-        "http": "varanus.client.transport.http.HttpTransport",
-        "https": "varanus.client.transport.http.HttpTransport",
+        "http": "varanus.client.transport.http.ThreadedHttpTransport",
+        "https": "varanus.client.transport.http.ThreadedHttpTransport",
         "db": "varanus.client.transport.database.ModelTransport",
     }
 
-    send_all: bool
     configured = False
 
     def setup(
         self,
-        dsn,
-        request_attr="varanus",
-        environment=None,
-        transport_class=None,
-        logger_name="varanus.request",
-        tags=None,
-        include_headers=None,
-        exclude_headers=None,
+        dsn: str,
+        environment: str,
+        node: str | None = None,
+        transport_class: str | type[BaseTransport] | None = None,
+        request_attr: str = "varanus",
+        logger_name: str = "varanus.request",
+        tags: dict | None = None,
+        include_headers: Iterable[str] | bool = False,
+        exclude_headers: Iterable[str] | None = None,
+        include_settings: Iterable[str] | bool = False,
+        exclude_settings: Iterable[str] | None = None,
+        include_default_settings: bool = False,
+        include_env: Iterable[str] | bool = False,
+        exclude_env: Iterable[str] | None = None,
         log_queries: bool | int = False,
-        log_query_params=False,
-        query_metrics=False,
-        send_all=False,
-        install=None,
+        log_query_params: bool = False,
+        log_query_stack: bool = False,
+        query_metrics: bool | str = False,
+        send_all: bool = False,
+        install: list | None = None,
     ):
         url = urlsplit(dsn)
         self.environment = environment
+        self.node = node or platform.node()
         if transport_class is None:
-            transport_class = self.scheme_transports.get(url.scheme)
-            if transport_class is None:
+            if url.scheme not in self.scheme_transports:
                 raise ValueError(f"No transport class found for `{url.scheme}`")
-        if isinstance(transport_class, str):
-            transport_class = import_string(transport_class)
-        self.transport = transport_class(url, self.environment)
+            resolved_class = import_string(self.scheme_transports[url.scheme])
+        elif isinstance(transport_class, str):
+            resolved_class = import_string(transport_class)
+        else:
+            resolved_class = transport_class
+        if not issubclass(resolved_class, BaseTransport):
+            raise ValueError(
+                f"Transport class `{transport_class}` must be a subclass of"
+                "BaseTransport."
+            )
+        self.transport = resolved_class(url, self.environment, self.node)
         self.request_attr = request_attr
         self.logger_name = logger_name
         self.tags = tags or {}
         self.include_headers = include_headers
-        self.exclude_headers = exclude_headers
+        self.exclude_headers = (
+            self.sensitive_headers if exclude_headers is None else set(exclude_headers)
+        )
+        self.include_settings = include_settings
+        self.exclude_settings = (
+            self.sensitive_settings
+            if exclude_settings is None
+            else set(exclude_settings)
+        )
+        self.include_default_settings = include_default_settings
+        self.include_env = include_env
+        self.exclude_env = (
+            self.sensitive_env if exclude_env is None else set(exclude_env)
+        )
         if log_queries or query_metrics:
             try:
                 # The logger is installed as early as possible, and for all connections.
@@ -86,6 +119,7 @@ class VaranusClient:
                 self.query_logger = QueryLogger(
                     log_queries,
                     log_query_params,
+                    log_query_stack,
                     query_metrics,
                 )
                 # Install it in each new connection (if it's not already installed).
@@ -98,10 +132,6 @@ class VaranusClient:
         self.send_all = send_all
         self.configured = True
         if install:
-            if not isinstance(install, list):
-                raise TypeError(
-                    "The varanus middleware can only be automatically installed into a list."
-                )
             if "django.contrib.auth.middleware.AuthenticationMiddleware" in install:
                 idx = install.index(
                     "django.contrib.auth.middleware.AuthenticationMiddleware"
@@ -119,16 +149,39 @@ class VaranusClient:
             self.transport.send(e)
 
     def ping(self):
+        from django import get_version
+        from django.conf import settings
+
+        include_settings = resolve_include_exclude(
+            [s for s in dir(settings) if s == s.upper()],
+            self.include_settings,
+            self.exclude_settings,
+        )
+        include_env = resolve_include_exclude(
+            os.environ.keys(),
+            self.include_env,
+            self.exclude_env,
+        )
+
         self.transport.ping(
             NodeInfo(
-                name=platform.node(),
+                name=self.node,
                 platform=platform.platform(),
-                python_version=platform.python_version(),
+                language=platform.python_implementation(),
+                language_version=platform.python_version(),
+                framework="Django",
+                framework_version=get_version(),
                 packages={d.name: d.version for d in distributions()},
+                settings={
+                    s: repr(getattr(settings, s))
+                    for s in include_settings
+                    if self.include_default_settings or settings.is_overridden(s)
+                },
+                environment={e: os.environ[e] for e in include_env},
             )
         )
 
-    def log(self, level, message, *args, **kwargs):
+    def log(self, level: int, message: str, *args, **kwargs):
         if ctx := current_context.get():
             kwargs.setdefault("stacklevel", 2)
             ctx.log(level, message, *args, **kwargs)
@@ -137,7 +190,7 @@ class VaranusClient:
         if ctx := current_context.get():
             ctx.raw_exception(exception, tags=tags)
 
-    def metric(self, name, value: float = 0.0, tags: dict | None = None):
+    def metric(self, name: str, value: float = 0.0, tags: dict | None = None):
         if ctx := current_context.get():
             ctx.metric(name, value, tags=tags)
 

varanus/client/context.py

@@ -78,11 +78,11 @@ class VaranusContext:
 
     def log(
         self,
-        level,
-        message,
+        level: int,
+        message: str,
         *args,
         exc_info=None,
-        stacklevel=1,
+        stacklevel: int = 1,
         tags: dict | None = None,
         **kwargs,
     ):
@@ -99,27 +99,27 @@ class VaranusContext:
             )
         )
 
-    def debug(self, message, *args, **kwargs):
+    def debug(self, message: str, *args, **kwargs):
         kwargs.setdefault("stacklevel", 2)
         self.log(logging.DEBUG, message, *args, **kwargs)
 
-    def info(self, message, *args, **kwargs):
+    def info(self, message: str, *args, **kwargs):
         kwargs.setdefault("stacklevel", 2)
         self.log(logging.INFO, message, *args, **kwargs)
 
-    def warning(self, message, *args, **kwargs):
+    def warning(self, message: str, *args, **kwargs):
         kwargs.setdefault("stacklevel", 2)
         self.log(logging.WARNING, message, *args, **kwargs)
 
-    def error(self, message, *args, **kwargs):
+    def error(self, message: str, *args, **kwargs):
         kwargs.setdefault("stacklevel", 2)
         self.log(logging.ERROR, message, *args, **kwargs)
 
-    def critical(self, message, *args, **kwargs):
+    def critical(self, message: str, *args, **kwargs):
         kwargs.setdefault("stacklevel", 2)
         self.log(logging.CRITICAL, message, *args, **kwargs)
 
-    def exception(self, message, *args, **kwargs):
+    def exception(self, message: str, *args, **kwargs):
         kwargs.setdefault("stacklevel", 2)
         kwargs.setdefault("exc_info", sys.exc_info())
         self.log(logging.ERROR, message, *args, **kwargs)
@@ -128,12 +128,12 @@ class VaranusContext:
         if err := Error.from_exception(exception, tags=tags):
             self.errors.append(err)
 
-    def context(self, name="", tags: dict | None = None):
+    def context(self, name: str = "", tags: dict | None = None):
         ctx = VaranusContext(self.client, name, tags)
         self.subcontexts.append(ctx)
         return ctx
 
-    def metric(self, name, value: float = 0.0, tags: dict | None = None):
+    def metric(self, name: str, value: float = 0.0, tags: dict | None = None):
         if name not in self.metrics:
             self.metrics[name] = Metric(name=name, tags=tags or {})
         self.metrics[name].update(value)

varanus/client/loggers.py

@@ -1,7 +1,7 @@
 import logging
 import sys
 
-from ..events import Log, Query, now
+from ..events import Log, Query, capture_stack, now
 from .context import ONE_MS, current_context
 
 
@@ -12,7 +12,7 @@ class VaranusHandler(logging.Handler):
 
 
 class QueryLogger:
-    def __init__(self, threshold, log_params, metrics):
+    def __init__(self, threshold, log_params, log_stack, metrics):
         # TODO: add callback for tagging?
         if threshold is True:
             self.threshold = 0
@@ -21,6 +21,7 @@ class QueryLogger:
         else:
             self.threshold = int(threshold)
         self.log_params = log_params
+        self.log_stack = log_stack
         if isinstance(metrics, str):
             self.metrics_name = metrics
         else:
@@ -55,5 +56,6 @@ class QueryLogger:
                         db=context["connection"].alias,
                         elapsed_ms=elapsed_ms,
                         success=success,
+                        stack=capture_stack(1) if self.log_stack else [],
                     )
                 )

varanus/client/middleware.py

@@ -1,12 +1,10 @@
-import re
+import ipaddress
 
 from django.core.exceptions import MiddlewareNotUsed
 from django.http import HttpRequest, HttpResponse
 
 from ..events import Request
-from .client import client
-
-IP_REGEX = re.compile(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}")
+from .client import client, resolve_include_exclude
 
 
 def get_ip(request: HttpRequest):
@@ -15,24 +13,21 @@ def get_ip(request: HttpRequest):
         ip_address = ip_address.split(",")[0].strip()
     if not ip_address:
         ip_address = request.META.get("REMOTE_ADDR", "127.0.0.1").strip()
-    if not IP_REGEX.match(ip_address):
-        ip_address = None
-    return ip_address
+    try:
+        # Validate and normalize the IP address.
+        return str(ipaddress.ip_address(ip_address))
+    except ValueError:
+        return ""
 
 
 def request_headers(request: HttpRequest):
     headers = {}
-    if not client.include_headers:
-        return headers
-    if client.include_headers is True:
-        include = set(name.lower() for name in request.headers)
-    else:
-        include = set(name.lower() for name in client.include_headers)
-    if client.exclude_headers is None:
-        exclude = client.sensitive_headers
-    else:
-        exclude = set(name.lower() for name in client.exclude_headers)
-    for name in sorted(include - exclude):
+    include_headers = resolve_include_exclude(
+        [name.lower() for name in request.headers],
+        client.include_headers,
+        client.exclude_headers,
+    )
+    for name in include_headers:
         value = request.headers.get(name)
         if value is not None:
             headers[name] = value
@@ -59,8 +54,9 @@ class VaranusMiddleware:
             # TODO: any need for request tags separate from context tags?
             varanus.request = Request(
                 host=request.get_host(),
-                path=request.path,
                 method=request.method or "",
+                path=request.path,
+                query=request.META.get("QUERY_STRING", ""),
                 status=response.status_code,
                 headers=request_headers(request),
                 size=(

varanus/client/transport/base.py

@@ -4,7 +4,7 @@ from varanus import events
 
 
 class BaseTransport:
-    def __init__(self, url: SplitResult, environment: str):
+    def __init__(self, url: SplitResult, environment: str, node: str):
         pass
 
     def ping(self, info: events.NodeInfo):

varanus/client/transport/database.py

@@ -1,28 +1,55 @@
+import typing
 import uuid
 from urllib.parse import SplitResult
 
-from django.db import transaction
-
 from varanus import events
-from varanus.server import models
 
 from .base import BaseTransport
 
+if typing.TYPE_CHECKING:
+    from varanus.server.models import Node, Site
+
 
 class ModelTransport(BaseTransport):
-    def __init__(self, url: SplitResult, environment: str):
+    site: "Site"
+    node: "Node"
+
+    def __init__(self, url: SplitResult, environment: str, node: str):
+        self.slug = url.netloc
         self.environment = environment
 
+    def ensure_site(self):
+        if hasattr(self, "site"):
+            return
+
+        from varanus.server.models import Site
+
+        self.site, created = Site.objects.get_or_create(
+            slug=self.slug,
+            defaults={
+                "name": self.slug,
+                "schema_name": self.slug,
+            },
+        )
+
     def ping(self, info: events.NodeInfo):
-        site = models.Site.objects.get()
-        models.Node.update(info, site=site, environment=self.environment)
+        from varanus.server.models import Node
+
+        self.ensure_site()
+
+        with self.site.activated():
+            self.node, created, updates = Node.update(
+                info, site=self.site, environment=self.environment
+            )
 
-    @transaction.atomic
     def send(self, event: events.Context):
-        site = models.Site.objects.get()
-        models.Context.from_event(
-            event,
-            event_id=uuid.uuid4(),
-            site=site,
-            environment=self.environment,
-        )
+        from varanus.server.models import Context
+
+        with self.site.activated():
+            Context.from_event(
+                event,
+                event_id=uuid.uuid4(),
+                site=self.site,
+                environment=self.environment,
+                node=self.node,
+            )

varanus/client/transport/http.py

@@ -1,6 +1,9 @@
-import platform
+import logging
+import queue
+import threading
+import time
 from typing import Any
-from urllib.parse import SplitResult
+from urllib.parse import SplitResult, parse_qs
 
 import httpx
 import msgspec
@@ -9,29 +12,72 @@ from varanus import events
 
 from .base import BaseTransport
 
+logger = logging.getLogger(__name__)
+
 
 class HttpTransport(BaseTransport):
-    def __init__(self, url: SplitResult, environment: str):
+    def __init__(self, url: SplitResult, environment: str, node: str):
         path = url.path.rstrip("/")
         self.ping_url = f"{url.scheme}://{url.netloc}{path}/api/ping/"
         self.event_url = f"{url.scheme}://{url.netloc}{path}/api/ingest/"
+        self.options = parse_qs(url.query, keep_blank_values=True)
+        timeout = 1.0
+        if "timeout" in self.options:
+            timeout = float(self.options["timeout"][0])
         self.client = httpx.Client(
             headers={
                 "X-Varanus-Key": url.username or "",
-                "X-Varanus-Environment": environment or "",
-                "X-Varanus-Node": platform.node(),
+                "X-Varanus-Environment": environment,
+                "X-Varanus-Node": node,
             },
-            timeout=1.0,
+            timeout=timeout,
         )
 
     def request(self, url: str, obj: Any):
         try:
             self.client.post(url, content=msgspec.json.encode(obj))
-        except Exception as ex:
-            print(f"error sending to {url}: {ex}")
+        except Exception:
+            logger.exception("error sending to %s", url)
 
     def ping(self, info: events.NodeInfo):
         self.request(self.ping_url, info)
 
     def send(self, event: events.Context):
-        self.request(self.event_url, event)
+        self.request(self.event_url, [event])
+
+
+def sender(pending: queue.SimpleQueue, client: httpx.Client, url: str, rate: float):
+    while True:
+        start = time.time()
+        events = []
+        while True:
+            try:
+                events.append(pending.get_nowait())
+            except queue.Empty:
+                break
+            if len(events) >= 100:
+                break
+        if events:
+            try:
+                client.post(url, content=msgspec.json.encode(events))
+            except Exception:
+                logger.exception("error sending to %s", url)
+        elapsed = time.time() - start
+        time.sleep(max(rate - elapsed, 1.0))
+
+
+class ThreadedHttpTransport(HttpTransport):
+    def __init__(self, url: SplitResult, environment: str, node: str):
+        super().__init__(url, environment, node)
+        self.pending = queue.SimpleQueue()
+        rate = 3.0
+        if "rate" in self.options:
+            rate = float(self.options["rate"][0])
+        threading.Thread(
+            target=sender,
+            args=(self.pending, self.client, self.event_url, rate),
+            daemon=True,
+        ).start()
+
+    def send(self, event: events.Context):
+        self.pending.put(event)

varanus/events.py

@@ -17,11 +17,16 @@ class Event(Struct, kw_only=True, omit_defaults=True):
 class NodeInfo(Struct):
     name: str
     platform: str
-    python_version: str
+    language: str
+    language_version: str
+    framework: str
+    framework_version: str
     packages: dict[str, str]
+    settings: dict[str, str]
+    environment: dict[str, str]
 
 
-class ErrorLine(Struct):
+class StackLine(Struct):
     file: str | None
     lineno: int | None
     function: str | None
@@ -30,11 +35,31 @@ class ErrorLine(Struct):
     locals: dict[str, str]
 
 
+def capture_stack(skip: int = 0, include_locals: bool = False) -> list[StackLine]:
+    lines = []
+    for frame in inspect.stack()[skip + 1 :]:
+        lines.append(
+            StackLine(
+                file=frame.filename,
+                lineno=frame.lineno,
+                linesrc=frame.code_context[0].strip() if frame.code_context else "",
+                function=frame.function,
+                module=frame.frame.f_globals.get("__name__", ""),
+                locals=(
+                    {name: repr(val) for name, val in frame.frame.f_locals.items()}
+                    if include_locals
+                    else {}
+                ),
+            )
+        )
+    return lines
+
+
 class Error(Event):
     kind: str
     module: str
     message: str
-    lines: list[ErrorLine] = []
+    stack: list[StackLine] = []
 
     @classmethod
     def from_exception(cls, exc_info, tags=None):
@@ -63,7 +88,7 @@ class Error(Event):
                     pass
             module = f_globals.get("__name__", "")
             lines.append(
-                ErrorLine(
+                StackLine(
                     file=abs_path,
                     lineno=lineno,
                     function=function,
@@ -78,7 +103,7 @@ class Error(Event):
             kind=kind,
             module=module,
             message=message,
-            lines=lines,
+            stack=lines,
         )
 
 
@@ -144,12 +169,14 @@ class Query(Event):
     db: str
     elapsed_ms: int
     success: bool
+    stack: list[StackLine] = []
 
 
 class Request(Event):
     host: str
-    path: str
     method: str
+    path: str
+    query: str
     status: int
     headers: dict = {}
     size: int | None = None

varanus/{server/search → search}/__init__.py

@@ -1,9 +1,10 @@
 from .base import Search
-from .date import DateRange
-from .facet import MultiFacet
+from .fields import DateRange, Filter, Hidden, MultiFacet
 
 __all__ = [
     "Search",
     "DateRange",
+    "Filter",
+    "Hidden",
     "MultiFacet",
 ]