analytics-platform 0.3.1__tar.gz → 0.3.3__tar.gz

{analytics_platform-0.3.1 → analytics_platform-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: analytics-platform
-Version: 0.3.1
+Version: 0.3.3
 Summary: Firewall, error monitoring and server analytics for your Python server, run from your analytics dashboard.
 Author: steviec87
 License-Expression: MIT
@@ -168,6 +168,11 @@ a few seconds later.
 What still can't be sent waits in a file in the temp folder and is sent when
 it can be, after a restart too; each error keeps the time it happened.
 
+An error in a request or job also shows the steps before it. Add them with
+`platform.add_breadcrumb("Order saved")`; a step added outside a request or job
+is dropped. Outgoing calls aren't recorded on their own, since Python has no
+public way to see them.
+
 Background jobs, as a decorator or a `with` block (sync or async):
 
 ```python
@@ -182,7 +187,10 @@ Several services on one site: `Platform(..., server_name="worker")`.
 **Several apps on one site** (microservices, a worker, a Node service next to
 this one): install the package in each, with the same `site_id`, `secret` and
 `endpoint`, and a `server_name` of its own. Errors, Server speed, Crawlers and
-Security → Updates then tell the apps apart. Without one, each app is named
+Security → Updates then tell the apps apart, and Health → All systems lists
+each with a light: the package checks in about once a minute, so an app that
+stops reads "Stopped" within ten minutes (`sdk/protocols/checkin.md`).
+Without one, each app is named
 after its machine: two apps on one machine are reported as one, and only the
 last package list sent is kept. A service only your other apps call takes
 `firewall=False, proxy=False, maintenance=False` (its requests all come from

{analytics_platform-0.3.1 → analytics_platform-0.3.3}/README.md

@@ -142,6 +142,11 @@ a few seconds later.
 What still can't be sent waits in a file in the temp folder and is sent when
 it can be, after a restart too; each error keeps the time it happened.
 
+An error in a request or job also shows the steps before it. Add them with
+`platform.add_breadcrumb("Order saved")`; a step added outside a request or job
+is dropped. Outgoing calls aren't recorded on their own, since Python has no
+public way to see them.
+
 Background jobs, as a decorator or a `with` block (sync or async):
 
 ```python
@@ -156,7 +161,10 @@ Several services on one site: `Platform(..., server_name="worker")`.
 **Several apps on one site** (microservices, a worker, a Node service next to
 this one): install the package in each, with the same `site_id`, `secret` and
 `endpoint`, and a `server_name` of its own. Errors, Server speed, Crawlers and
-Security → Updates then tell the apps apart. Without one, each app is named
+Security → Updates then tell the apps apart, and Health → All systems lists
+each with a light: the package checks in about once a minute, so an app that
+stops reads "Stopped" within ten minutes (`sdk/protocols/checkin.md`).
+Without one, each app is named
 after its machine: two apps on one machine are reported as one, and only the
 last package list sent is kept. A service only your other apps call takes
 `firewall=False, proxy=False, maintenance=False` (its requests all come from

{analytics_platform-0.3.1 → analytics_platform-0.3.3}/analytics_platform/__init__.py

@@ -18,7 +18,7 @@ import functools
 import time
 
 from .core import Core, SESSION_HEADER, VERSION, header, request_target
-from .errors import current_request
+from .errors import add_breadcrumb, current_request, current_trail
 
 __all__ = ["create_platform", "Platform", "PlatformMiddleware", "VERSION"]
 
@@ -81,7 +81,7 @@ class Platform:
                  performance=True, crawlers=True, forms=True, headers=True, audit=True, flags=True, maintenance=True,
                  inventory=True,
                  posture=True, resources=True, proxy=True, server_name=None):
-        self.core = Core(site_id, secret, endpoint, debug)
+        self.core = Core(site_id, secret, endpoint, debug, server_name=server_name)
         if not self.core.valid:
             print("[analytics-platform] site_id, secret and endpoint are required; nothing is sent and every request passes")
         settings = {"firewall": firewall, "errors": errors, "analytics": analytics, "performance": performance,
@@ -154,6 +154,12 @@ class Platform:
         if errors:
             errors.capture_exception(error, request=request, status=status, kind=kind)
 
+    def add_breadcrumb(self, text):
+        """A step of your own in the trail shown before an error in this
+        request or job."""
+        if self._modules.get("errors"):
+            add_breadcrumb(text)
+
     def job(self, name):
         """A background job (a queue consumer, a cron task), as a decorator or
         a with block: its errors are sent with the job's name as their route,
@@ -365,6 +371,7 @@ class Platform:
                 response_headers = list(response_headers) + platform.security_headers(n for n, _ in response_headers) + remember
                 return start_response(status, response_headers, exc_info) if exc_info else start_response(status, response_headers)
 
+            trail = current_trail.set([])
             try:
                 return app(environ, timed_start_response)
             except Exception as error:
@@ -374,6 +381,7 @@ class Platform:
                 })
                 raise
             finally:
+                current_trail.reset(trail)
                 if timing:
                     platform.time_request(environ.get("REQUEST_METHOD", ""), "", seen["status"], started, environ.get("PATH_INFO", ""))
 
@@ -453,18 +461,21 @@ class _Job:
         return {"method": "JOB", "path": self.route, "route": self.route, "session_id": ""}
 
     def __enter__(self):
-        self._runs.append((time.perf_counter(), current_request.set(self._fields)))
+        self._runs.append((time.perf_counter(), current_request.set(self._fields), current_trail.set([])))
         return self
 
     def __exit__(self, kind, value, tb):
-        started, token = self._runs.pop()
+        started, token, trail = self._runs.pop()
         current_request.reset(token)
-        if value is not None and not isinstance(value, Exception):
-            return False  # cancelled or interrupted (CancelledError, KeyboardInterrupt): not a run that failed
-        if value is not None:
-            self.platform.capture_exception(value, status=500, kind="error", request=self._fields())
-        self.platform.time_request("JOB", self.route, 500 if value is not None else 200, started, self.route)
-        return False
+        try:
+            if value is not None and not isinstance(value, Exception):
+                return False  # cancelled or interrupted (CancelledError, KeyboardInterrupt): not a run that failed
+            if value is not None:
+                self.platform.capture_exception(value, status=500, kind="error", request=self._fields())
+            self.platform.time_request("JOB", self.route, 500 if value is not None else 200, started, self.route)
+            return False
+        finally:
+            current_trail.reset(trail)
 
     async def __aenter__(self):
         return self.__enter__()
@@ -556,6 +567,7 @@ class PlatformMiddleware:
             "method": scope.get("method", ""), "path": scope.get("path", ""),
             "route": getattr(scope.get("route"), "path", ""), "session_id": headers.get(SESSION_HEADER, ""),
         })
+        trail = current_trail.set([])
         try:
             await self.app(scope, receive, timed_send)
         except Exception as error:
@@ -566,6 +578,7 @@ class PlatformMiddleware:
             raise
         finally:
             current_request.reset(token)
+            current_trail.reset(trail)
             # The router sets scope["route"] (FastAPI, Starlette) on the way in.
             self.platform.time_request(scope.get("method", ""), getattr(scope.get("route"), "path", "") or "",
                                        seen["status"], started, scope.get("path", ""))

{analytics_platform-0.3.1 → analytics_platform-0.3.3}/analytics_platform/core.py

@@ -10,13 +10,16 @@ start again in a forked worker (gunicorn, uWSGI), where threads don't survive.
 import atexit
 import json
 import os
+import platform
 import re
+import socket
 import threading
+import time
 import urllib.error
 import urllib.parse
 import urllib.request
 
-VERSION = "0.3.1"
+VERSION = "0.3.3"
 TIMEOUT = 5
 MIN_SYNC = 30
 RETRY = 60
@@ -100,9 +103,30 @@ class Periodic:
             thread.join(wait)
 
 
+def _rss_mb():
+    """This process's resident memory in MB, where the platform says (Linux
+    /proc; elsewhere the peak, from resource); None when neither can."""
+    try:
+        with open("/proc/self/statm") as statm:
+            return round(int(statm.read().split()[1]) * os.sysconf("SC_PAGE_SIZE") / 1048576)
+    except Exception:
+        pass
+    try:
+        import resource
+        import sys
+        peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+        return round(peak / 1048576 if sys.platform == "darwin" else peak / 1024)
+    except Exception:
+        return None
+
+
 class Core:
-    def __init__(self, site_id, secret, endpoint, debug=False):
+    def __init__(self, site_id, secret, endpoint, debug=False, server_name=None):
         self.site_id = str(site_id or "")
+        # The name this service checks in under (X-Platform-App, on the switches
+        # request): server_name, as every module sends it, else the machine's.
+        self.server_name = str(server_name or socket.gethostname())[:100]
+        self._started = time.time()
         self.secret = str(secret or "")
         self.endpoint = str(endpoint or "").rstrip("/")
         self.valid = bool(re.fullmatch(r"[a-fA-F0-9]{24}", self.site_id) and self.secret and self.endpoint)
@@ -154,8 +178,25 @@ class Core:
         """False only when the dashboard turned the module off."""
         return self._switches.get(name) is not False
 
+    def _check_in(self):
+        """This service's check-in, form-encoded so any name is plain ASCII:
+        Health -> All systems shows it running and when it was last heard from.
+        None when it can't be built: the switches still sync, without it."""
+        try:
+            fields = {"server": self.server_name, "kind": "process", "pid": os.getpid(),
+                      "up_s": int(time.time() - self._started), "runtime": "Python " + platform.python_version()}
+            rss = _rss_mb()
+            if rss is not None:
+                fields["rss_mb"] = rss
+            return urllib.parse.urlencode(fields) if self.server_name else None
+        except Exception:
+            return None
+
     def _sync_switches(self):
         headers = {"X-Platform-Modules": ",".join(self._loaded)}
+        app = self._check_in()
+        if app:
+            headers["X-Platform-App"] = app
         if self._etag:
             headers["If-None-Match"] = self._etag
         res = self.send("core", "/events/platform/config?site_id=" + self.site_id, method="GET", auth="bearer", headers=headers)

{analytics_platform-0.3.1 → analytics_platform-0.3.3}/analytics_platform/django.py

@@ -16,7 +16,7 @@ from django.conf import settings
 from django.http import HttpResponse
 
 from . import SESSION_HEADER, Platform
-from .errors import current_request
+from .errors import current_request, current_trail
 
 _platform = None
 
@@ -96,10 +96,12 @@ class PlatformMiddleware:
         # Errors captured during the request (logged, or caught without the
         # request) still know which request they came from.
         token = current_request.set(lambda: _request_fields(request))
+        trail = current_trail.set([])
         try:
             response = self.get_response(request)
         finally:
             current_request.reset(token)
+            current_trail.reset(trail)
         for name, value in self.platform.security_headers(response.headers.keys()):
             response[name] = value
         if closed:

{analytics_platform-0.3.1 → analytics_platform-0.3.3}/analytics_platform/errors.py

@@ -8,7 +8,8 @@ error is sent once however many of those see it. Errors are sent in small
 batches, a batch the collector couldn't take is tried once more, and sending
 never raises. What still can't be sent (the collector is away, or the process
 is crashing) waits in a file and is sent once it can be, after a restart too;
-each error carries when it happened.
+each error carries when it happened, and the trail of its request or job: the
+steps you add with add_breadcrumb().
 """
 import contextvars
 import hashlib
@@ -44,6 +45,25 @@ SPOOL_BYTES = 1024 * 1024
 # returning {"method", "path", "route", "session_id"}, so an error captured
 # without a request (logged, or caught deep in your code) still gets its route.
 current_request = contextvars.ContextVar("analytics_platform_request", default=None)
+# The steps of the request or job being handled, oldest first, set to a new
+# list wherever current_request starts a request. Outside one, steps are dropped.
+current_trail = contextvars.ContextVar("analytics_platform_trail", default=None)
+CRUMB_MAX = 15
+
+
+def add_breadcrumb(text):
+    trail = current_trail.get()
+    if trail is None:
+        return
+    trail.append({"type": "note", "text": str(text)[:200], "at": time.time()})
+    del trail[:-CRUMB_MAX]
+
+
+def _breadcrumbs():
+    """The trail as the tracker sends it: how long before now each step happened."""
+    now = time.time()
+    return [{"type": crumb["type"], "text": crumb["text"], "ms": max(0, round((now - crumb["at"]) * 1000))}
+            for crumb in current_trail.get() or []]
 
 
 def _request_fields():
@@ -172,6 +192,9 @@ class Errors:
                 # When it happened: a crash sent after a restart keeps its own time.
                 "at": datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z"),
             }
+            trail = _breadcrumbs()
+            if trail:
+                event["breadcrumbs"] = trail
             event.update({k: str(v or "") for k, v in (request or {}).items() if k in ("method", "path", "route", "session_id")})
             if event.get("path"):
                 event["path"] = event["path"].split("?")[0]

{analytics_platform-0.3.1 → analytics_platform-0.3.3}/analytics_platform.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: analytics-platform
-Version: 0.3.1
+Version: 0.3.3
 Summary: Firewall, error monitoring and server analytics for your Python server, run from your analytics dashboard.
 Author: steviec87
 License-Expression: MIT
@@ -168,6 +168,11 @@ a few seconds later.
 What still can't be sent waits in a file in the temp folder and is sent when
 it can be, after a restart too; each error keeps the time it happened.
 
+An error in a request or job also shows the steps before it. Add them with
+`platform.add_breadcrumb("Order saved")`; a step added outside a request or job
+is dropped. Outgoing calls aren't recorded on their own, since Python has no
+public way to see them.
+
 Background jobs, as a decorator or a `with` block (sync or async):
 
 ```python
@@ -182,7 +187,10 @@ Several services on one site: `Platform(..., server_name="worker")`.
 **Several apps on one site** (microservices, a worker, a Node service next to
 this one): install the package in each, with the same `site_id`, `secret` and
 `endpoint`, and a `server_name` of its own. Errors, Server speed, Crawlers and
-Security → Updates then tell the apps apart. Without one, each app is named
+Security → Updates then tell the apps apart, and Health → All systems lists
+each with a light: the package checks in about once a minute, so an app that
+stops reads "Stopped" within ten minutes (`sdk/protocols/checkin.md`).
+Without one, each app is named
 after its machine: two apps on one machine are reported as one, and only the
 last package list sent is kept. A service only your other apps call takes
 `firewall=False, proxy=False, maintenance=False` (its requests all come from

{analytics_platform-0.3.1 → analytics_platform-0.3.3}/analytics_platform.egg-info/SOURCES.txt

@@ -22,6 +22,7 @@ analytics_platform.egg-info/PKG-INFO
 analytics_platform.egg-info/SOURCES.txt
 analytics_platform.egg-info/dependency_links.txt
 analytics_platform.egg-info/top_level.txt
+tests/test_checkin.py
 tests/test_crawlers.py
 tests/test_errors.py
 tests/test_firewall.py

{analytics_platform-0.3.1 → analytics_platform-0.3.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "analytics-platform"
-version = "0.3.1"
+version = "0.3.3"
 description = "Firewall, error monitoring and server analytics for your Python server, run from your analytics dashboard."
 readme = "README.md"
 requires-python = ">=3.8"

analytics_platform-0.3.3/tests/test_checkin.py

@@ -0,0 +1,29 @@
+"""The check-in on the switches request (X-Platform-App): what names this
+service on Health -> All systems."""
+import os
+import unittest
+import urllib.parse
+from unittest import mock
+
+from analytics_platform.core import Core
+
+
+class CheckInTest(unittest.TestCase):
+    def test_switches_request_says_which_service_asks_and_that_it_runs(self):
+        core = Core("a" * 24, "s", "http://collector", server_name="api wörker")
+        with mock.patch.object(core, "send", return_value=None) as send:
+            core._sync_switches()
+        header = send.call_args.kwargs["headers"]["X-Platform-App"]
+        fields = dict(urllib.parse.parse_qsl(header))
+        self.assertEqual(fields["server"], "api wörker")
+        self.assertEqual(fields["kind"], "process")
+        self.assertEqual(fields["pid"], str(os.getpid()))
+        self.assertTrue(fields["runtime"].startswith("Python "))
+        self.assertTrue(header.isascii())
+
+    def test_default_name_is_the_machines(self):
+        self.assertTrue(Core("a" * 24, "s", "http://collector").server_name)
+
+
+if __name__ == "__main__":
+    unittest.main()

{analytics_platform-0.3.1 → analytics_platform-0.3.3}/tests/test_errors.py

@@ -13,7 +13,7 @@ import unittest
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
 
 from analytics_platform import errors as mod  # noqa: E402
-from analytics_platform.errors import Errors, current_request  # noqa: E402
+from analytics_platform.errors import Errors, add_breadcrumb, current_request, current_trail  # noqa: E402
 
 
 class Answer:
@@ -249,5 +249,51 @@ class ErrorsTest(unittest.TestCase):
         kept = open(errors.spool_file).read().splitlines()
         self.assertEqual([json.loads(line)["message"] for line in kept], ["e10", "e11"], "once, and not what was sent")
 
+
+class Breadcrumbs(unittest.TestCase):
+    def capture(self, errors, message):
+        errors.capture_exception(ValueError(message))
+
+    def test_an_error_carries_the_last_steps_of_its_request(self):
+        core = FakeCore()
+        errors = Errors(core, capture_uncaught=False)
+        add_breadcrumb("outside a request, kept nowhere")
+        self.capture(errors, "no trail")
+        token = current_trail.set([])
+        try:
+            for i in range(20):
+                add_breadcrumb("step %d" % i)
+            self.capture(errors, "after the steps")
+        finally:
+            current_trail.reset(token)
+        self.capture(errors, "no trail either")
+        errors.flush()
+        first, second, third = core.sent[0]
+        self.assertNotIn("breadcrumbs", first)
+        self.assertNotIn("breadcrumbs", third)
+        crumbs = second["breadcrumbs"]
+        self.assertEqual(len(crumbs), 15)
+        self.assertEqual(crumbs[0]["text"], "step 5")
+        self.assertEqual(crumbs[-1]["type"], "note")
+        self.assertNotIn("at", crumbs[0])
+        self.assertGreaterEqual(crumbs[0]["ms"], 0)
+
+    def test_a_job_has_its_own_trail_and_keeps_it_for_the_error_it_raises(self):
+        from analytics_platform import create_platform
+        platform = create_platform(site_id="a1b2c3d4e5f6a1b2c3d4e5f6", secret="s", endpoint="http://127.0.0.1:9",
+                                   firewall=False, analytics=False, flags=False, inventory=False, posture=False,
+                                   resources=False, proxy=False, maintenance=False, crawlers=False, forms=False,
+                                   audit=False, errors={"capture_uncaught": False, "spool_dir": False})
+        seen = []
+        platform._modules["errors"].before_send = lambda event, error: seen.append(event) or None
+        with self.assertRaises(ValueError):
+            with platform.job("send-receipts"):
+                platform.add_breadcrumb("Order saved")
+                raise ValueError("in a job")
+        self.assertEqual([crumb["text"] for crumb in seen[0]["breadcrumbs"]], ["Order saved"])
+        self.assertIsNone(current_trail.get(), "the trail ends with the job")
+        platform.close()
+
+
 if __name__ == "__main__":
     unittest.main()