analytics-platform 0.3.0__tar.gz → 0.3.2__tar.gz

{analytics_platform-0.3.0 → analytics_platform-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: analytics-platform
-Version: 0.3.0
+Version: 0.3.2
 Summary: Firewall, error monitoring and server analytics for your Python server, run from your analytics dashboard.
 Author: steviec87
 License-Expression: MIT
@@ -168,6 +168,11 @@ a few seconds later.
 What still can't be sent waits in a file in the temp folder and is sent when
 it can be, after a restart too; each error keeps the time it happened.
 
+An error in a request or job also shows the steps before it. Add them with
+`platform.add_breadcrumb("Order saved")`; a step added outside a request or job
+is dropped. Outgoing calls aren't recorded on their own, since Python has no
+public way to see them.
+
 Background jobs, as a decorator or a `with` block (sync or async):
 
 ```python
@@ -179,6 +184,16 @@ Their errors carry the job's name as the route, a job that raises is sent
 (status 500) and raised on, and each run is timed (Server speed → Jobs).
 Several services on one site: `Platform(..., server_name="worker")`.
 
+**Several apps on one site** (microservices, a worker, a Node service next to
+this one): install the package in each, with the same `site_id`, `secret` and
+`endpoint`, and a `server_name` of its own. Errors, Server speed, Crawlers and
+Security → Updates then tell the apps apart. Without one, each app is named
+after its machine: two apps on one machine are reported as one, and only the
+last package list sent is kept. A service only your other apps call takes
+`firewall=False, proxy=False, maintenance=False` (its requests all come from
+your own apps, so the rate limit would see one visitor); on a machine shared
+by several apps, keep `resources` on in only one of them.
+
 Options: `server_name`, `root` (traceback paths relative to it),
 `capture_uncaught`, `capture_logging`, `spool_dir` (where unsent errors wait;
 `False` for none), `before_send(event, error)` (change an error, or return

{analytics_platform-0.3.0 → analytics_platform-0.3.2}/README.md

@@ -142,6 +142,11 @@ a few seconds later.
 What still can't be sent waits in a file in the temp folder and is sent when
 it can be, after a restart too; each error keeps the time it happened.
 
+An error in a request or job also shows the steps before it. Add them with
+`platform.add_breadcrumb("Order saved")`; a step added outside a request or job
+is dropped. Outgoing calls aren't recorded on their own, since Python has no
+public way to see them.
+
 Background jobs, as a decorator or a `with` block (sync or async):
 
 ```python
@@ -153,6 +158,16 @@ Their errors carry the job's name as the route, a job that raises is sent
 (status 500) and raised on, and each run is timed (Server speed → Jobs).
 Several services on one site: `Platform(..., server_name="worker")`.
 
+**Several apps on one site** (microservices, a worker, a Node service next to
+this one): install the package in each, with the same `site_id`, `secret` and
+`endpoint`, and a `server_name` of its own. Errors, Server speed, Crawlers and
+Security → Updates then tell the apps apart. Without one, each app is named
+after its machine: two apps on one machine are reported as one, and only the
+last package list sent is kept. A service only your other apps call takes
+`firewall=False, proxy=False, maintenance=False` (its requests all come from
+your own apps, so the rate limit would see one visitor); on a machine shared
+by several apps, keep `resources` on in only one of them.
+
 Options: `server_name`, `root` (traceback paths relative to it),
 `capture_uncaught`, `capture_logging`, `spool_dir` (where unsent errors wait;
 `False` for none), `before_send(event, error)` (change an error, or return

{analytics_platform-0.3.0 → analytics_platform-0.3.2}/analytics_platform/__init__.py

@@ -18,7 +18,7 @@ import functools
 import time
 
 from .core import Core, SESSION_HEADER, VERSION, header, request_target
-from .errors import current_request
+from .errors import add_breadcrumb, current_request, current_trail
 
 __all__ = ["create_platform", "Platform", "PlatformMiddleware", "VERSION"]
 
@@ -100,6 +100,10 @@ class Platform:
             # so its errors, timings and resources can be told apart.
             if server_name and name in ("errors", "performance", "crawlers", "resources"):
                 opts.setdefault("server_name", server_name)
+            # Inventory and posture call it host, and are stored one per site
+            # and host: without it two services on one machine overwrite each other's.
+            if server_name and name in ("inventory", "posture"):
+                opts.setdefault("host", server_name)
             try:
                 self._modules[name] = _load(name, self.core, opts)
             except Exception as error:
@@ -150,6 +154,12 @@ class Platform:
         if errors:
             errors.capture_exception(error, request=request, status=status, kind=kind)
 
+    def add_breadcrumb(self, text):
+        """A step of your own in the trail shown before an error in this
+        request or job."""
+        if self._modules.get("errors"):
+            add_breadcrumb(text)
+
     def job(self, name):
         """A background job (a queue consumer, a cron task), as a decorator or
         a with block: its errors are sent with the job's name as their route,
@@ -361,6 +371,7 @@ class Platform:
                 response_headers = list(response_headers) + platform.security_headers(n for n, _ in response_headers) + remember
                 return start_response(status, response_headers, exc_info) if exc_info else start_response(status, response_headers)
 
+            trail = current_trail.set([])
             try:
                 return app(environ, timed_start_response)
             except Exception as error:
@@ -370,6 +381,7 @@ class Platform:
                 })
                 raise
             finally:
+                current_trail.reset(trail)
                 if timing:
                     platform.time_request(environ.get("REQUEST_METHOD", ""), "", seen["status"], started, environ.get("PATH_INFO", ""))
 
@@ -449,18 +461,21 @@ class _Job:
         return {"method": "JOB", "path": self.route, "route": self.route, "session_id": ""}
 
     def __enter__(self):
-        self._runs.append((time.perf_counter(), current_request.set(self._fields)))
+        self._runs.append((time.perf_counter(), current_request.set(self._fields), current_trail.set([])))
         return self
 
     def __exit__(self, kind, value, tb):
-        started, token = self._runs.pop()
+        started, token, trail = self._runs.pop()
         current_request.reset(token)
-        if value is not None and not isinstance(value, Exception):
-            return False  # cancelled or interrupted (CancelledError, KeyboardInterrupt): not a run that failed
-        if value is not None:
-            self.platform.capture_exception(value, status=500, kind="error", request=self._fields())
-        self.platform.time_request("JOB", self.route, 500 if value is not None else 200, started, self.route)
-        return False
+        try:
+            if value is not None and not isinstance(value, Exception):
+                return False  # cancelled or interrupted (CancelledError, KeyboardInterrupt): not a run that failed
+            if value is not None:
+                self.platform.capture_exception(value, status=500, kind="error", request=self._fields())
+            self.platform.time_request("JOB", self.route, 500 if value is not None else 200, started, self.route)
+            return False
+        finally:
+            current_trail.reset(trail)
 
     async def __aenter__(self):
         return self.__enter__()
@@ -552,6 +567,7 @@ class PlatformMiddleware:
             "method": scope.get("method", ""), "path": scope.get("path", ""),
             "route": getattr(scope.get("route"), "path", ""), "session_id": headers.get(SESSION_HEADER, ""),
         })
+        trail = current_trail.set([])
         try:
             await self.app(scope, receive, timed_send)
         except Exception as error:
@@ -562,6 +578,7 @@ class PlatformMiddleware:
             raise
         finally:
             current_request.reset(token)
+            current_trail.reset(trail)
             # The router sets scope["route"] (FastAPI, Starlette) on the way in.
             self.platform.time_request(scope.get("method", ""), getattr(scope.get("route"), "path", "") or "",
                                        seen["status"], started, scope.get("path", ""))

{analytics_platform-0.3.0 → analytics_platform-0.3.2}/analytics_platform/core.py

@@ -16,7 +16,7 @@ import urllib.error
 import urllib.parse
 import urllib.request
 
-VERSION = "0.3.0"
+VERSION = "0.3.2"
 TIMEOUT = 5
 MIN_SYNC = 30
 RETRY = 60

{analytics_platform-0.3.0 → analytics_platform-0.3.2}/analytics_platform/django.py

@@ -16,7 +16,7 @@ from django.conf import settings
 from django.http import HttpResponse
 
 from . import SESSION_HEADER, Platform
-from .errors import current_request
+from .errors import current_request, current_trail
 
 _platform = None
 
@@ -96,10 +96,12 @@ class PlatformMiddleware:
         # Errors captured during the request (logged, or caught without the
         # request) still know which request they came from.
         token = current_request.set(lambda: _request_fields(request))
+        trail = current_trail.set([])
         try:
             response = self.get_response(request)
         finally:
             current_request.reset(token)
+            current_trail.reset(trail)
         for name, value in self.platform.security_headers(response.headers.keys()):
             response[name] = value
         if closed:

{analytics_platform-0.3.0 → analytics_platform-0.3.2}/analytics_platform/errors.py

@@ -8,7 +8,8 @@ error is sent once however many of those see it. Errors are sent in small
 batches, a batch the collector couldn't take is tried once more, and sending
 never raises. What still can't be sent (the collector is away, or the process
 is crashing) waits in a file and is sent once it can be, after a restart too;
-each error carries when it happened.
+each error carries when it happened, and the trail of its request or job: the
+steps you add with add_breadcrumb().
 """
 import contextvars
 import hashlib
@@ -44,6 +45,25 @@ SPOOL_BYTES = 1024 * 1024
 # returning {"method", "path", "route", "session_id"}, so an error captured
 # without a request (logged, or caught deep in your code) still gets its route.
 current_request = contextvars.ContextVar("analytics_platform_request", default=None)
+# The steps of the request or job being handled, oldest first, set to a new
+# list wherever current_request starts a request. Outside one, steps are dropped.
+current_trail = contextvars.ContextVar("analytics_platform_trail", default=None)
+CRUMB_MAX = 15
+
+
+def add_breadcrumb(text):
+    trail = current_trail.get()
+    if trail is None:
+        return
+    trail.append({"type": "note", "text": str(text)[:200], "at": time.time()})
+    del trail[:-CRUMB_MAX]
+
+
+def _breadcrumbs():
+    """The trail as the tracker sends it: how long before now each step happened."""
+    now = time.time()
+    return [{"type": crumb["type"], "text": crumb["text"], "ms": max(0, round((now - crumb["at"]) * 1000))}
+            for crumb in current_trail.get() or []]
 
 
 def _request_fields():
@@ -172,6 +192,9 @@ class Errors:
                 # When it happened: a crash sent after a restart keeps its own time.
                 "at": datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z"),
             }
+            trail = _breadcrumbs()
+            if trail:
+                event["breadcrumbs"] = trail
             event.update({k: str(v or "") for k, v in (request or {}).items() if k in ("method", "path", "route", "session_id")})
             if event.get("path"):
                 event["path"] = event["path"].split("?")[0]

{analytics_platform-0.3.0 → analytics_platform-0.3.2}/analytics_platform/proxy.py

@@ -47,6 +47,13 @@ def prefix_for(secret):
     return hashlib.sha256(("proxy|" + str(secret)).encode("utf-8")).hexdigest()[:8]
 
 
+# Sent as X-Proxy-Key, so the collector believes the visitor addresses this
+# proxy passes on: anyone can write X-Forwarded-For. Worked out from the secret
+# as the collector does (expresscollect/utils/proxyKey.js).
+def key_for(secret):
+    return hashlib.sha256(("proxykey|" + str(secret)).encode("utf-8")).hexdigest()[:32]
+
+
 class Proxy:
     """prefix: the path it answers on (default: from the secret)."""
 
@@ -54,6 +61,7 @@ class Proxy:
         self.core = core
         self.enabled = enabled
         self.prefix = re.sub(r"[^A-Za-z0-9_-]", "", str(prefix or prefix_for(core.secret)))
+        self.key = key_for(core.secret) if core.secret else ""
         # One copy of each tracker file, so a repeat visitor costs no upstream
         # request. A handful of small files: it never grows.
         self._files = {}
@@ -139,6 +147,8 @@ class Proxy:
         # The visitor's address: the collector needs it for the country, the
         # daily visitor ID and the per-visitor rate limit.
         out = {"X-Forwarded-For": client_ip or ""}
+        if self.key:
+            out["X-Proxy-Key"] = self.key
         for name in PASS_HEADERS:
             value = headers.get(name)
             if value:

{analytics_platform-0.3.0 → analytics_platform-0.3.2}/analytics_platform.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: analytics-platform
-Version: 0.3.0
+Version: 0.3.2
 Summary: Firewall, error monitoring and server analytics for your Python server, run from your analytics dashboard.
 Author: steviec87
 License-Expression: MIT
@@ -168,6 +168,11 @@ a few seconds later.
 What still can't be sent waits in a file in the temp folder and is sent when
 it can be, after a restart too; each error keeps the time it happened.
 
+An error in a request or job also shows the steps before it. Add them with
+`platform.add_breadcrumb("Order saved")`; a step added outside a request or job
+is dropped. Outgoing calls aren't recorded on their own, since Python has no
+public way to see them.
+
 Background jobs, as a decorator or a `with` block (sync or async):
 
 ```python
@@ -179,6 +184,16 @@ Their errors carry the job's name as the route, a job that raises is sent
 (status 500) and raised on, and each run is timed (Server speed → Jobs).
 Several services on one site: `Platform(..., server_name="worker")`.
 
+**Several apps on one site** (microservices, a worker, a Node service next to
+this one): install the package in each, with the same `site_id`, `secret` and
+`endpoint`, and a `server_name` of its own. Errors, Server speed, Crawlers and
+Security → Updates then tell the apps apart. Without one, each app is named
+after its machine: two apps on one machine are reported as one, and only the
+last package list sent is kept. A service only your other apps call takes
+`firewall=False, proxy=False, maintenance=False` (its requests all come from
+your own apps, so the rate limit would see one visitor); on a machine shared
+by several apps, keep `resources` on in only one of them.
+
 Options: `server_name`, `root` (traceback paths relative to it),
 `capture_uncaught`, `capture_logging`, `spool_dir` (where unsent errors wait;
 `False` for none), `before_send(event, error)` (change an error, or return

{analytics_platform-0.3.0 → analytics_platform-0.3.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "analytics-platform"
-version = "0.3.0"
+version = "0.3.2"
 description = "Firewall, error monitoring and server analytics for your Python server, run from your analytics dashboard."
 readme = "README.md"
 requires-python = ">=3.8"

{analytics_platform-0.3.0 → analytics_platform-0.3.2}/tests/test_errors.py

@@ -13,7 +13,7 @@ import unittest
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
 
 from analytics_platform import errors as mod  # noqa: E402
-from analytics_platform.errors import Errors, current_request  # noqa: E402
+from analytics_platform.errors import Errors, add_breadcrumb, current_request, current_trail  # noqa: E402
 
 
 class Answer:
@@ -249,5 +249,51 @@ class ErrorsTest(unittest.TestCase):
         kept = open(errors.spool_file).read().splitlines()
         self.assertEqual([json.loads(line)["message"] for line in kept], ["e10", "e11"], "once, and not what was sent")
 
+
+class Breadcrumbs(unittest.TestCase):
+    def capture(self, errors, message):
+        errors.capture_exception(ValueError(message))
+
+    def test_an_error_carries_the_last_steps_of_its_request(self):
+        core = FakeCore()
+        errors = Errors(core, capture_uncaught=False)
+        add_breadcrumb("outside a request, kept nowhere")
+        self.capture(errors, "no trail")
+        token = current_trail.set([])
+        try:
+            for i in range(20):
+                add_breadcrumb("step %d" % i)
+            self.capture(errors, "after the steps")
+        finally:
+            current_trail.reset(token)
+        self.capture(errors, "no trail either")
+        errors.flush()
+        first, second, third = core.sent[0]
+        self.assertNotIn("breadcrumbs", first)
+        self.assertNotIn("breadcrumbs", third)
+        crumbs = second["breadcrumbs"]
+        self.assertEqual(len(crumbs), 15)
+        self.assertEqual(crumbs[0]["text"], "step 5")
+        self.assertEqual(crumbs[-1]["type"], "note")
+        self.assertNotIn("at", crumbs[0])
+        self.assertGreaterEqual(crumbs[0]["ms"], 0)
+
+    def test_a_job_has_its_own_trail_and_keeps_it_for_the_error_it_raises(self):
+        from analytics_platform import create_platform
+        platform = create_platform(site_id="a1b2c3d4e5f6a1b2c3d4e5f6", secret="s", endpoint="http://127.0.0.1:9",
+                                   firewall=False, analytics=False, flags=False, inventory=False, posture=False,
+                                   resources=False, proxy=False, maintenance=False, crawlers=False, forms=False,
+                                   audit=False, errors={"capture_uncaught": False, "spool_dir": False})
+        seen = []
+        platform._modules["errors"].before_send = lambda event, error: seen.append(event) or None
+        with self.assertRaises(ValueError):
+            with platform.job("send-receipts"):
+                platform.add_breadcrumb("Order saved")
+                raise ValueError("in a job")
+        self.assertEqual([crumb["text"] for crumb in seen[0]["breadcrumbs"]], ["Order saved"])
+        self.assertIsNone(current_trail.get(), "the trail ends with the job")
+        platform.close()
+
+
 if __name__ == "__main__":
     unittest.main()

{analytics_platform-0.3.0 → analytics_platform-0.3.2}/tests/test_proxy.py

@@ -12,7 +12,7 @@ from http.server import BaseHTTPRequestHandler, HTTPServer
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
 
-from analytics_platform.proxy import Proxy, prefix_for  # noqa: E402
+from analytics_platform.proxy import Proxy, key_for, prefix_for  # noqa: E402
 
 TRACKER = b"// a tracker" + b"x" * 600
 
@@ -77,6 +77,10 @@ class ProxyTest(unittest.TestCase):
         Collector.seen = []
         self.proxy = Proxy(FakeCore(self.endpoint))
 
+    def test_the_proxy_key_is_worked_out_as_the_collector_does(self):
+        # The same vector is pinned in the collector, the backend and the other packages.
+        self.assertEqual(key_for("0123456789abcdef01234567"), "8fc74dbbd4f088449862841cd29fcaee")
+
     def test_the_address_comes_from_the_secret(self):
         self.assertEqual(prefix_for("s3cret"), prefix_for("s3cret"))
         self.assertNotEqual(prefix_for("s3cret"), prefix_for("other"))
@@ -123,6 +127,7 @@ class ProxyTest(unittest.TestCase):
         self.assertEqual(got["path"], "/collectanalytics/e?v=1")
         self.assertEqual(got["body"], b'{"event":"pageview"}')
         self.assertEqual(got["headers"].get("X-Forwarded-For"), "198.51.100.4")
+        self.assertEqual(got["headers"].get("X-Proxy-Key"), key_for("s3cret"), "so the collector believes the address")
         self.assertEqual(got["headers"].get("User-Agent"), "Mozilla/5.0")
         self.assertIsNone(got["headers"].get("Cookie"), "the site's own cookies stay on the site")