analytics-platform 0.2.1__tar.gz → 0.3.0__tar.gz

{analytics_platform-0.2.1 → analytics_platform-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: analytics-platform
-Version: 0.2.1
+Version: 0.3.0
 Summary: Firewall, error monitoring and server analytics for your Python server, run from your analytics dashboard.
 Author: steviec87
 License-Expression: MIT
@@ -103,7 +103,7 @@ create_platform(site_id=..., secret=..., endpoint=...,
                 analytics=False)
 ```
 
-**Switch modules off in the dashboard** (Security → Settings → Server package) without a
+**Switch modules off in the dashboard** (Security → Server package) without a
 deploy: the package checks about once a minute. Modules are independent: one
 failing to start, or the collector being down, never stops the others, and
 never slows your requests. Background work runs in daemon threads, which start
@@ -158,8 +158,30 @@ except PaymentError as error:
     raise
 ```
 
+Exceptions your code only logs: `errors={"capture_logging": True}` sends an
+exception logged at ERROR or above (`logger.exception(...)`, or
+`exc_info=True`); it is still logged. An exception is sent once however many
+of these see it, and one logged inside a Django, Flask or ASGI request keeps
+that request's route. A batch the collector couldn't take is tried once more
+a few seconds later.
+
+What still can't be sent waits in a file in the temp folder and is sent when
+it can be, after a restart too; each error keeps the time it happened.
+
+Background jobs, as a decorator or a `with` block (sync or async):
+
+```python
+@platform.job("send-receipts")
+def send_receipts(): ...
+```
+
+Their errors carry the job's name as the route, a job that raises is sent
+(status 500) and raised on, and each run is timed (Server speed → Jobs).
+Several services on one site: `Platform(..., server_name="worker")`.
+
 Options: `server_name`, `root` (traceback paths relative to it),
-`capture_uncaught`, `before_send(event, error)` (change an error, or return
+`capture_uncaught`, `capture_logging`, `spool_dir` (where unsent errors wait;
+`False` for none), `before_send(event, error)` (change an error, or return
 `None` to drop it).
 
 ## Server analytics

{analytics_platform-0.2.1 → analytics_platform-0.3.0}/README.md

@@ -77,7 +77,7 @@ create_platform(site_id=..., secret=..., endpoint=...,
                 analytics=False)
 ```
 
-**Switch modules off in the dashboard** (Security → Settings → Server package) without a
+**Switch modules off in the dashboard** (Security → Server package) without a
 deploy: the package checks about once a minute. Modules are independent: one
 failing to start, or the collector being down, never stops the others, and
 never slows your requests. Background work runs in daemon threads, which start
@@ -132,8 +132,30 @@ except PaymentError as error:
     raise
 ```
 
+Exceptions your code only logs: `errors={"capture_logging": True}` sends an
+exception logged at ERROR or above (`logger.exception(...)`, or
+`exc_info=True`); it is still logged. An exception is sent once however many
+of these see it, and one logged inside a Django, Flask or ASGI request keeps
+that request's route. A batch the collector couldn't take is tried once more
+a few seconds later.
+
+What still can't be sent waits in a file in the temp folder and is sent when
+it can be, after a restart too; each error keeps the time it happened.
+
+Background jobs, as a decorator or a `with` block (sync or async):
+
+```python
+@platform.job("send-receipts")
+def send_receipts(): ...
+```
+
+Their errors carry the job's name as the route, a job that raises is sent
+(status 500) and raised on, and each run is timed (Server speed → Jobs).
+Several services on one site: `Platform(..., server_name="worker")`.
+
 Options: `server_name`, `root` (traceback paths relative to it),
-`capture_uncaught`, `before_send(event, error)` (change an error, or return
+`capture_uncaught`, `capture_logging`, `spool_dir` (where unsent errors wait;
+`False` for none), `before_send(event, error)` (change an error, or return
 `None` to drop it).
 
 ## Server analytics

{analytics_platform-0.2.1 → analytics_platform-0.3.0}/analytics_platform/__init__.py

@@ -14,9 +14,11 @@ A module is on with True (or its options as a dict) and off with False.
 No dependencies; Python 3.8+.
 """
 import asyncio
+import functools
 import time
 
-from .core import Core, SESSION_HEADER, VERSION, header
+from .core import Core, SESSION_HEADER, VERSION, header, request_target
+from .errors import current_request
 
 __all__ = ["create_platform", "Platform", "PlatformMiddleware", "VERSION"]
 
@@ -78,7 +80,7 @@ class Platform:
     def __init__(self, site_id, secret, endpoint, trust_proxy=None, debug=False, firewall=True, errors=True, analytics=True,
                  performance=True, crawlers=True, forms=True, headers=True, audit=True, flags=True, maintenance=True,
                  inventory=True,
-                 posture=True, resources=True, proxy=True):
+                 posture=True, resources=True, proxy=True, server_name=None):
         self.core = Core(site_id, secret, endpoint, debug)
         if not self.core.valid:
             print("[analytics-platform] site_id, secret and endpoint are required; nothing is sent and every request passes")
@@ -94,6 +96,10 @@ class Platform:
             opts = dict(setting) if isinstance(setting, dict) else {}
             if name == "firewall":
                 opts.setdefault("trust_proxy", trust_proxy)
+            # server_name names this service for every module that sends one,
+            # so its errors, timings and resources can be told apart.
+            if server_name and name in ("errors", "performance", "crawlers", "resources"):
+                opts.setdefault("server_name", server_name)
             try:
                 self._modules[name] = _load(name, self.core, opts)
             except Exception as error:
@@ -144,6 +150,19 @@ class Platform:
         if errors:
             errors.capture_exception(error, request=request, status=status, kind=kind)
 
+    def job(self, name):
+        """A background job (a queue consumer, a cron task), as a decorator or
+        a with block: its errors are sent with the job's name as their route,
+        and each run is timed (Health → Server speed → Jobs); a run that raises
+        counts as failed, and the exception is raised on.
+
+            @platform.job("send-receipts")
+            def send_receipts(): ...
+
+            with platform.job("nightly-import"): ...
+        """
+        return _Job(self, name)
+
     def login_failed(self, request_or_ip):
         """A failed login (bot protection): the address, or a Django, Flask or
         Starlette request (its address as trust_proxy says)."""
@@ -286,7 +305,8 @@ class Platform:
 
     def blocked(self, remote, headers, method, path):
         """(status, body, headers) when the firewall turns the request away,
-        else None. headers: a dict with lowercase names."""
+        else None. headers: a dict with lowercase names. path: the request
+        target as sent, query string and all, not yet decoded (request_target)."""
         firewall = self._modules.get("firewall")
         if not firewall:
             return None
@@ -321,7 +341,7 @@ class Platform:
                 start_response("%d " % status, list(extra))
                 return [body]
             blocked = platform.blocked(environ.get("REMOTE_ADDR", ""), headers, environ.get("REQUEST_METHOD", ""),
-                                       environ.get("PATH_INFO", ""))
+                                       _wsgi_target(environ))
             if blocked:
                 status, body, extra = blocked
                 start_response("%d %s" % (status, body.decode()), extra)
@@ -387,6 +407,17 @@ class Platform:
         return PlatformMiddleware(app, platform=self)
 
 
+def _wsgi_target(environ):
+    # gunicorn keeps the target as sent (RAW_URI), uWSGI and mod_wsgi too
+    # (REQUEST_URI); PATH_INFO is decoded, and PEP 3333 hands its bytes over as latin-1.
+    path = environ.get("SCRIPT_NAME", "") + environ.get("PATH_INFO", "")
+    try:
+        path = path.encode("latin-1")
+    except UnicodeEncodeError:
+        pass
+    return request_target(path, environ.get("QUERY_STRING", ""), environ.get("RAW_URI") or environ.get("REQUEST_URI"))
+
+
 def _wsgi_body(environ):
     try:
         length = int(environ.get("CONTENT_LENGTH") or 0)
@@ -405,6 +436,53 @@ class _Headers:
         pass
 
 
+class _Job:
+    """platform.job(name): a with block (sync or async) or a decorator."""
+
+    def __init__(self, platform, name):
+        self.platform = platform
+        self.name = name
+        self.route = "/(job) " + str(name or "job")[:180]
+        self._runs = []
+
+    def _fields(self):
+        return {"method": "JOB", "path": self.route, "route": self.route, "session_id": ""}
+
+    def __enter__(self):
+        self._runs.append((time.perf_counter(), current_request.set(self._fields)))
+        return self
+
+    def __exit__(self, kind, value, tb):
+        started, token = self._runs.pop()
+        current_request.reset(token)
+        if value is not None and not isinstance(value, Exception):
+            return False  # cancelled or interrupted (CancelledError, KeyboardInterrupt): not a run that failed
+        if value is not None:
+            self.platform.capture_exception(value, status=500, kind="error", request=self._fields())
+        self.platform.time_request("JOB", self.route, 500 if value is not None else 200, started, self.route)
+        return False
+
+    async def __aenter__(self):
+        return self.__enter__()
+
+    async def __aexit__(self, kind, value, tb):
+        return self.__exit__(kind, value, tb)
+
+    def __call__(self, fn):
+        if asyncio.iscoroutinefunction(fn):
+            @functools.wraps(fn)
+            async def run_async(*args, **kwargs):
+                with _Job(self.platform, self.name):
+                    return await fn(*args, **kwargs)
+            return run_async
+
+        @functools.wraps(fn)
+        def run(*args, **kwargs):
+            with _Job(self.platform, self.name):
+                return fn(*args, **kwargs)
+        return run
+
+
 class PlatformMiddleware:
     """ASGI middleware: app.add_middleware(PlatformMiddleware, platform=platform)."""
 
@@ -439,7 +517,9 @@ class PlatformMiddleware:
                         "headers": [(k.lower().encode(), str(v).encode()) for k, v in extra]})
             await send({"type": "http.response.body", "body": out})
             return
-        blocked = self.platform.blocked(client[0] or "", headers, scope.get("method", ""), scope.get("path", ""))
+        # raw_path is the path as sent; servers that leave it out get path encoded again.
+        target = request_target(scope.get("path", ""), scope.get("query_string", b""), scope.get("raw_path"))
+        blocked = self.platform.blocked(client[0] or "", headers, scope.get("method", ""), target)
         if blocked:
             status, body, extra = blocked
             await send({"type": "http.response.start", "status": status,
@@ -465,6 +545,13 @@ class PlatformMiddleware:
                     message = dict(message, headers=list(message.get("headers") or []) + [(n.lower().encode(), v.encode("latin-1", "replace")) for n, v in extra] + remember)
             await send(message)
 
+        # Errors captured during the request (logged, or caught without the
+        # request) still know which request they came from. The router sets
+        # scope["route"] on the way in, so it is read when the error is.
+        token = current_request.set(lambda: {
+            "method": scope.get("method", ""), "path": scope.get("path", ""),
+            "route": getattr(scope.get("route"), "path", ""), "session_id": headers.get(SESSION_HEADER, ""),
+        })
         try:
             await self.app(scope, receive, timed_send)
         except Exception as error:
@@ -474,6 +561,7 @@ class PlatformMiddleware:
             })
             raise
         finally:
+            current_request.reset(token)
             # The router sets scope["route"] (FastAPI, Starlette) on the way in.
             self.platform.time_request(scope.get("method", ""), getattr(scope.get("route"), "path", "") or "",
                                        seen["status"], started, scope.get("path", ""))

{analytics_platform-0.2.1 → analytics_platform-0.3.0}/analytics_platform/core.py

@@ -13,9 +13,10 @@ import os
 import re
 import threading
 import urllib.error
+import urllib.parse
 import urllib.request
 
-VERSION = "0.1.0"
+VERSION = "0.3.0"
 TIMEOUT = 5
 MIN_SYNC = 30
 RETRY = 60
@@ -197,3 +198,27 @@ def header(headers, name):
     """A header from a dict with lowercase names (or None)."""
     value = headers.get(name.lower())
     return value.strip() if isinstance(value, str) else ""
+
+
+# The characters a path may carry unencoded (RFC 3986), which quote() leaves alone.
+_PATH_SAFE = "/:@!$&'()*+,;=-._~"
+
+
+def request_target(path, query="", raw=None):
+    """The request target as the visitor sent it, query string and all, which
+    is what the request rules match (sdk/protocols/firewall.md): the Node and
+    PHP clients hand over req.url and REQUEST_URI. raw: the server's own copy
+    of it when it keeps one. Otherwise the path the framework has already
+    decoded is encoded again, so the rules' one decoding pass is the only one.
+    path: text, or the bytes it was decoded from."""
+    if isinstance(raw, bytes):
+        raw = raw.decode("latin-1")
+    if isinstance(raw, str) and raw:
+        target = raw
+    else:
+        target = urllib.parse.quote(path or "", safe=_PATH_SAFE)
+    if isinstance(query, bytes):
+        query = query.decode("latin-1")
+    if query and "?" not in target:
+        target += "?" + query
+    return target

{analytics_platform-0.2.1 → analytics_platform-0.3.0}/analytics_platform/django.py

@@ -16,6 +16,7 @@ from django.conf import settings
 from django.http import HttpResponse
 
 from . import SESSION_HEADER, Platform
+from .errors import current_request
 
 _platform = None
 
@@ -74,7 +75,9 @@ class PlatformMiddleware:
             for name, value in extra:
                 response[name] = value
             return response
-        blocked = self.platform.blocked(ip, headers, request.method, request.path)
+        # The target as sent: request.path is decoded, get_full_path() encodes it again.
+        target = request.META.get("RAW_URI") or request.META.get("REQUEST_URI") or request.get_full_path()
+        blocked = self.platform.blocked(ip, headers, request.method, target)
         if blocked:
             status, body, extra = blocked
             response = HttpResponse(body, status=status)
@@ -90,23 +93,34 @@ class PlatformMiddleware:
             for name, value in closed[1]:
                 response[name] = value
             return response
-        response = self.get_response(request)
+        # Errors captured during the request (logged, or caught without the
+        # request) still know which request they came from.
+        token = current_request.set(lambda: _request_fields(request))
+        try:
+            response = self.get_response(request)
+        finally:
+            current_request.reset(token)
         for name, value in self.platform.security_headers(response.headers.keys()):
             response[name] = value
         if closed:
             response["Set-Cookie"] = closed[1][0][1]
-        match = getattr(request, "resolver_match", None)
-        route = getattr(match, "route", "") or ""
-        self.platform.time_request(request.method, ("/" + route.lstrip("^").rstrip("$").lstrip("/")) if route else "",
-                                   response.status_code, started, request.path)
+        self.platform.time_request(request.method, _route(request), response.status_code, started, request.path)
         self.platform.record_crawler(headers.get("user-agent", ""), request.path, response.status_code, started,
                                      response.headers.get("Content-Length") or 0, ip)
         return response
 
     def process_exception(self, request, exception):
-        match = getattr(request, "resolver_match", None)
-        self.platform.capture_exception(exception, status=500, kind="error", request={
-            "method": request.method, "path": request.path, "route": getattr(match, "route", "") or "",
-            "session_id": request.headers.get(SESSION_HEADER, ""),
-        })
+        self.platform.capture_exception(exception, status=500, kind="error", request=_request_fields(request))
         return None
+
+
+# The URL pattern, as Server speed names it, so an error's route and the
+# route's timings match.
+def _route(request):
+    route = getattr(getattr(request, "resolver_match", None), "route", "") or ""
+    return ("/" + route.lstrip("^").rstrip("$").lstrip("/")) if route else ""
+
+
+def _request_fields(request):
+    return {"method": request.method, "path": request.path, "route": _route(request),
+            "session_id": request.headers.get(SESSION_HEADER, "")}