varanus 0.1.0.dev2__tar.gz → 0.1.0.dev3__tar.gz

{varanus-0.1.0.dev2 → varanus-0.1.0.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: varanus
-Version: 0.1.0.dev2
+Version: 0.1.0.dev3
 Summary: Django application monitoring.
 Requires-Dist: httpx>=0.27.0
 Requires-Dist: msgspec>=0.19.0

{varanus-0.1.0.dev2 → varanus-0.1.0.dev3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "varanus"
-version = "0.1.0.dev2"
+version = "0.1.0.dev3"
 description = "Django application monitoring."
 readme = "README.md"
 requires-python = ">=3.11"

{varanus-0.1.0.dev2 → varanus-0.1.0.dev3}/src/varanus/client/apps.py

@@ -5,4 +5,4 @@ class VaranusClient(AppConfig):
     name = "varanus.client"
 
     def ready(self):
-        print("VaranusClient.ready")
+        pass

{varanus-0.1.0.dev2 → varanus-0.1.0.dev3}/src/varanus/client/client.py

@@ -1,6 +1,5 @@
 import platform
 from importlib.metadata import distributions
-from typing import Union
 from urllib.parse import urlsplit
 
 from varanus.events import Context, NodeInfo
@@ -20,14 +19,14 @@ def install_query_logger(logger):
 
 
 class VaranusClient:
-    environment: str | None
+    environment: str
     transport: BaseTransport
 
     request_attr: str
     logger_name: str
     tags: dict
 
-    include_headers: Union[list, bool, None]
+    include_headers: list | bool | None
     exclude_headers: list | None
     sensitive_headers = set(
         [
@@ -38,9 +37,8 @@ class VaranusClient:
     )
 
     scheme_transports = {
-        "test": "varanus.client.transport.test.TestTransport",
-        "http": "varanus.client.transport.http.HttpTransport",
-        "https": "varanus.client.transport.http.HttpTransport",
+        "http": "varanus.client.transport.http.ThreadedHttpTransport",
+        "https": "varanus.client.transport.http.ThreadedHttpTransport",
         "db": "varanus.client.transport.database.ModelTransport",
     }
 
@@ -49,29 +47,38 @@ class VaranusClient:
 
     def setup(
         self,
-        dsn,
-        request_attr="varanus",
-        environment=None,
-        transport_class=None,
-        logger_name="varanus.request",
-        tags=None,
-        include_headers=None,
-        exclude_headers=None,
+        dsn: str,
+        node: str | None = None,
+        environment: str = "",
+        transport_class: str | type[BaseTransport] | None = None,
+        request_attr: str = "varanus",
+        logger_name: str = "varanus.request",
+        tags: dict | None = None,
+        include_headers: list | bool | None = None,
+        exclude_headers: list | None = None,
         log_queries: bool | int = False,
-        log_query_params=False,
-        query_metrics=False,
-        send_all=False,
-        install=None,
+        log_query_params: bool = False,
+        query_metrics: bool | str = False,
+        send_all: bool = False,
+        install: list | None = None,
     ):
         url = urlsplit(dsn)
         self.environment = environment
+        self.node = node or platform.node()
         if transport_class is None:
-            transport_class = self.scheme_transports.get(url.scheme)
-            if transport_class is None:
+            if url.scheme not in self.scheme_transports:
                 raise ValueError(f"No transport class found for `{url.scheme}`")
-        if isinstance(transport_class, str):
-            transport_class = import_string(transport_class)
-        self.transport = transport_class(url, self.environment)
+            resolved_class = import_string(self.scheme_transports[url.scheme])
+        elif isinstance(transport_class, str):
+            resolved_class = import_string(transport_class)
+        else:
+            resolved_class = transport_class
+        if not issubclass(resolved_class, BaseTransport):
+            raise ValueError(
+                f"Transport class `{transport_class}` must be a subclass of"
+                "BaseTransport."
+            )
+        self.transport = resolved_class(url, self.environment, self.node)
         self.request_attr = request_attr
         self.logger_name = logger_name
         self.tags = tags or {}
@@ -98,10 +105,6 @@ class VaranusClient:
         self.send_all = send_all
         self.configured = True
         if install:
-            if not isinstance(install, list):
-                raise TypeError(
-                    "The varanus middleware can only be automatically installed into a list."
-                )
             if "django.contrib.auth.middleware.AuthenticationMiddleware" in install:
                 idx = install.index(
                     "django.contrib.auth.middleware.AuthenticationMiddleware"
@@ -121,14 +124,14 @@ class VaranusClient:
     def ping(self):
         self.transport.ping(
             NodeInfo(
-                name=platform.node(),
+                name=self.node,
                 platform=platform.platform(),
                 python_version=platform.python_version(),
                 packages={d.name: d.version for d in distributions()},
             )
         )
 
-    def log(self, level, message, *args, **kwargs):
+    def log(self, level: int, message: str, *args, **kwargs):
         if ctx := current_context.get():
             kwargs.setdefault("stacklevel", 2)
             ctx.log(level, message, *args, **kwargs)
@@ -137,7 +140,7 @@ class VaranusClient:
         if ctx := current_context.get():
             ctx.raw_exception(exception, tags=tags)
 
-    def metric(self, name, value: float = 0.0, tags: dict | None = None):
+    def metric(self, name: str, value: float = 0.0, tags: dict | None = None):
         if ctx := current_context.get():
             ctx.metric(name, value, tags=tags)
 

{varanus-0.1.0.dev2 → varanus-0.1.0.dev3}/src/varanus/client/context.py

@@ -78,11 +78,11 @@ class VaranusContext:
 
     def log(
         self,
-        level,
-        message,
+        level: int,
+        message: str,
         *args,
         exc_info=None,
-        stacklevel=1,
+        stacklevel: int = 1,
         tags: dict | None = None,
         **kwargs,
     ):
@@ -99,27 +99,27 @@ class VaranusContext:
             )
         )
 
-    def debug(self, message, *args, **kwargs):
+    def debug(self, message: str, *args, **kwargs):
         kwargs.setdefault("stacklevel", 2)
         self.log(logging.DEBUG, message, *args, **kwargs)
 
-    def info(self, message, *args, **kwargs):
+    def info(self, message: str, *args, **kwargs):
         kwargs.setdefault("stacklevel", 2)
         self.log(logging.INFO, message, *args, **kwargs)
 
-    def warning(self, message, *args, **kwargs):
+    def warning(self, message: str, *args, **kwargs):
         kwargs.setdefault("stacklevel", 2)
         self.log(logging.WARNING, message, *args, **kwargs)
 
-    def error(self, message, *args, **kwargs):
+    def error(self, message: str, *args, **kwargs):
         kwargs.setdefault("stacklevel", 2)
         self.log(logging.ERROR, message, *args, **kwargs)
 
-    def critical(self, message, *args, **kwargs):
+    def critical(self, message: str, *args, **kwargs):
         kwargs.setdefault("stacklevel", 2)
         self.log(logging.CRITICAL, message, *args, **kwargs)
 
-    def exception(self, message, *args, **kwargs):
+    def exception(self, message: str, *args, **kwargs):
         kwargs.setdefault("stacklevel", 2)
         kwargs.setdefault("exc_info", sys.exc_info())
         self.log(logging.ERROR, message, *args, **kwargs)
@@ -128,12 +128,12 @@ class VaranusContext:
         if err := Error.from_exception(exception, tags=tags):
             self.errors.append(err)
 
-    def context(self, name="", tags: dict | None = None):
+    def context(self, name: str = "", tags: dict | None = None):
         ctx = VaranusContext(self.client, name, tags)
         self.subcontexts.append(ctx)
         return ctx
 
-    def metric(self, name, value: float = 0.0, tags: dict | None = None):
+    def metric(self, name: str, value: float = 0.0, tags: dict | None = None):
         if name not in self.metrics:
             self.metrics[name] = Metric(name=name, tags=tags or {})
         self.metrics[name].update(value)

{varanus-0.1.0.dev2 → varanus-0.1.0.dev3}/src/varanus/client/middleware.py

@@ -1,4 +1,4 @@
-import re
+import ipaddress
 
 from django.core.exceptions import MiddlewareNotUsed
 from django.http import HttpRequest, HttpResponse
@@ -6,8 +6,6 @@ from django.http import HttpRequest, HttpResponse
 from ..events import Request
 from .client import client
 
-IP_REGEX = re.compile(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}")
-
 
 def get_ip(request: HttpRequest):
     ip_address = request.META.get("HTTP_X_FORWARDED_FOR", "").strip()
@@ -15,9 +13,11 @@ def get_ip(request: HttpRequest):
         ip_address = ip_address.split(",")[0].strip()
     if not ip_address:
         ip_address = request.META.get("REMOTE_ADDR", "127.0.0.1").strip()
-    if not IP_REGEX.match(ip_address):
-        ip_address = None
-    return ip_address
+    try:
+        # Validate and normalize the IP address.
+        return str(ipaddress.ip_address(ip_address))
+    except ValueError:
+        return ""
 
 
 def request_headers(request: HttpRequest):

{varanus-0.1.0.dev2 → varanus-0.1.0.dev3}/src/varanus/client/transport/base.py

@@ -4,7 +4,7 @@ from varanus import events
 
 
 class BaseTransport:
-    def __init__(self, url: SplitResult, environment: str):
+    def __init__(self, url: SplitResult, environment: str, node: str):
         pass
 
     def ping(self, info: events.NodeInfo):

{varanus-0.1.0.dev2 → varanus-0.1.0.dev3}/src/varanus/client/transport/database.py

@@ -10,8 +10,9 @@ from .base import BaseTransport
 
 
 class ModelTransport(BaseTransport):
-    def __init__(self, url: SplitResult, environment: str):
+    def __init__(self, url: SplitResult, environment: str, node: str):
         self.environment = environment
+        self.node = node
 
     def ping(self, info: events.NodeInfo):
         site = models.Site.objects.get()
@@ -20,9 +21,11 @@ class ModelTransport(BaseTransport):
     @transaction.atomic
     def send(self, event: events.Context):
         site = models.Site.objects.get()
+        node = site.nodes.get(name=self.node, environment=self.environment)
         models.Context.from_event(
             event,
             event_id=uuid.uuid4(),
             site=site,
             environment=self.environment,
+            node=node,
         )

varanus-0.1.0.dev3/src/varanus/client/transport/http.py

@@ -0,0 +1,83 @@
+import logging
+import queue
+import threading
+import time
+from typing import Any
+from urllib.parse import SplitResult, parse_qs
+
+import httpx
+import msgspec
+
+from varanus import events
+
+from .base import BaseTransport
+
+logger = logging.getLogger(__name__)
+
+
+class HttpTransport(BaseTransport):
+    def __init__(self, url: SplitResult, environment: str, node: str):
+        path = url.path.rstrip("/")
+        self.ping_url = f"{url.scheme}://{url.netloc}{path}/api/ping/"
+        self.event_url = f"{url.scheme}://{url.netloc}{path}/api/ingest/"
+        self.options = parse_qs(url.query, keep_blank_values=True)
+        timeout = 1.0
+        if "timeout" in self.options:
+            timeout = float(self.options["timeout"][0])
+        self.client = httpx.Client(
+            headers={
+                "X-Varanus-Key": url.username or "",
+                "X-Varanus-Environment": environment,
+                "X-Varanus-Node": node,
+            },
+            timeout=timeout,
+        )
+
+    def request(self, url: str, obj: Any):
+        try:
+            self.client.post(url, content=msgspec.json.encode(obj))
+        except Exception:
+            logger.exception("error sending to %s", url)
+
+    def ping(self, info: events.NodeInfo):
+        self.request(self.ping_url, info)
+
+    def send(self, event: events.Context):
+        self.request(self.event_url, [event])
+
+
+def sender(pending: queue.SimpleQueue, client: httpx.Client, url: str, rate: float):
+    while True:
+        start = time.time()
+        events = []
+        while True:
+            try:
+                events.append(pending.get_nowait())
+            except queue.Empty:
+                break
+            if len(events) >= 100:
+                break
+        if events:
+            try:
+                client.post(url, content=msgspec.json.encode(events))
+            except Exception:
+                logger.exception("error sending to %s", url)
+        elapsed = time.time() - start
+        time.sleep(max(rate - elapsed, 1.0))
+
+
+class ThreadedHttpTransport(HttpTransport):
+    def __init__(self, url: SplitResult, environment: str, node: str):
+        super().__init__(url, environment, node)
+        self.pending = queue.SimpleQueue()
+        rate = 3.0
+        if "rate" in self.options:
+            rate = float(self.options["rate"][0])
+        threading.Thread(
+            target=sender,
+            args=(self.pending, self.client, self.event_url, rate),
+            daemon=True,
+        ).start()
+
+    def send(self, event: events.Context):
+        self.pending.put(event)

{varanus-0.1.0.dev2 → varanus-0.1.0.dev3}/src/varanus/server/search/base.py

@@ -46,7 +46,6 @@ class SearchField:
         self.order = order
 
     def __set_name__(self, owner, name: str):
-        # print("__set_name__", owner, name)
         assert issubclass(owner, Search)
         self.name = name
         self.prefix = f"{name}_"
@@ -60,7 +59,6 @@ class SearchField:
         owner._fields.append(self)
 
     def __get__(self, instance: "Search", owner=None) -> StringValues | Self:
-        # print("__get__", instance, owner)
         if owner is None:
             return self
         return {
@@ -70,7 +68,7 @@ class SearchField:
         }
 
     def __set__(self, instance, value):
-        print("__set__", instance, value)
+        raise AttributeError(f"`{self.name}` is read-only")
 
     def apply(self, queryset: QuerySet, field_data: StringValues) -> QuerySet:
         raise NotImplementedError()

{varanus-0.1.0.dev2 → varanus-0.1.0.dev3}/src/varanus/server/views/api.py

@@ -22,6 +22,7 @@ class APIView(View):
             self.key = SiteKey.objects.select_related("site").get(
                 access_key=request.META.get("HTTP_X_VARANUS_KEY", ""), active=True
             )
+            self.node = request.META.get("HTTP_X_VARANUS_NODE", "")
             self.environment = request.META.get("HTTP_X_VARANUS_ENVIRONMENT", "")
             self.site = self.key.site
             with self.site.activated():
@@ -32,7 +33,6 @@ class APIView(View):
         except SiteKey.DoesNotExist:
             return JsonResponse({"error": "Invalid site key."}, status=401)
         except Exception as e:
-            raise e
             return JsonResponse({"error": str(e)}, status=400)
 
 
@@ -45,17 +45,25 @@ class NodePingView(APIView):
 
 class IngestView(APIView):
     def post(self, request, *args, **kwargs):
-        event = msgspec.json.decode(request.body, type=events.Context)
+        try:
+            # Try to decode a list of events first.
+            event_list = msgspec.json.decode(request.body, type=list[events.Context])
+        except msgspec.ValidationError:
+            # Fall back to single event decoding.
+            event_list = [msgspec.json.decode(request.body, type=events.Context)]
         # TODO: cache these
         node = self.site.nodes.filter(
-            name=request.META.get("HTTP_X_VARANUS_NODE", ""),
-            environment=self.environment,
-        ).first()
-        context = Context.from_event(
-            event,
-            event_id=uuid.uuid4(),
-            site=self.site,
+            name=self.node,
             environment=self.environment,
-            node=node,
-        )
-        return {"event_id": context.event_id}
+        ).get()
+        created_ids = []
+        for event in event_list:
+            context = Context.from_event(
+                event,
+                event_id=uuid.uuid4(),
+                site=self.site,
+                environment=self.environment,
+                node=node,
+            )
+            created_ids.append(context.event_id)
+        return {"created": created_ids}

varanus-0.1.0.dev2/src/varanus/client/transport/http.py

@@ -1,37 +0,0 @@
-import platform
-from typing import Any
-from urllib.parse import SplitResult
-
-import httpx
-import msgspec
-
-from varanus import events
-
-from .base import BaseTransport
-
-
-class HttpTransport(BaseTransport):
-    def __init__(self, url: SplitResult, environment: str):
-        path = url.path.rstrip("/")
-        self.ping_url = f"{url.scheme}://{url.netloc}{path}/api/ping/"
-        self.event_url = f"{url.scheme}://{url.netloc}{path}/api/ingest/"
-        self.client = httpx.Client(
-            headers={
-                "X-Varanus-Key": url.username or "",
-                "X-Varanus-Environment": environment or "",
-                "X-Varanus-Node": platform.node(),
-            },
-            timeout=1.0,
-        )
-
-    def request(self, url: str, obj: Any):
-        try:
-            self.client.post(url, content=msgspec.json.encode(obj))
-        except Exception as ex:
-            print(f"error sending to {url}: {ex}")
-
-    def ping(self, info: events.NodeInfo):
-        self.request(self.ping_url, info)
-
-    def send(self, event: events.Context):
-        self.request(self.event_url, event)