analytics-platform 0.2.1__py3-none-any.whl

analytics_platform/core.py

@@ -0,0 +1,199 @@
+"""What every module shares: the site's credentials, sending to the
+collector, logging, background jobs, and the dashboard's module switches
+(Site settings -> Server, GET /events/platform/config about once a minute).
+
+A module the dashboard turns off stays loaded but does nothing; until the
+switches arrive, or when they can't be fetched, every loaded module stays on.
+Sending never raises into the app. Background jobs are daemon threads that
+start again in a forked worker (gunicorn, uWSGI), where threads don't survive.
+"""
+import atexit
+import json
+import os
+import re
+import threading
+import urllib.error
+import urllib.request
+
+VERSION = "0.1.0"
+TIMEOUT = 5
+MIN_SYNC = 30
+RETRY = 60
+SESSION_HEADER = "x-analytics-session"
+
+
+class Response:
+    def __init__(self, status, headers, body):
+        self.status = status
+        self.headers = headers
+        self.body = body
+
+    @property
+    def ok(self):
+        return 200 <= self.status < 300
+
+    def json(self):
+        try:
+            return json.loads(self.body or b"null")
+        except ValueError:
+            return None
+
+
+_running = []
+
+
+def _stop_all():
+    """At exit: background threads finish before Python shuts down (a daemon
+    thread still running then can crash the interpreter)."""
+    for job in list(_running):
+        job.stop()
+
+
+atexit.register(_stop_all)
+
+
+class Periodic:
+    """Runs job() in a daemon thread; job returns the seconds until its next run."""
+
+    def __init__(self, job, first=0.0):
+        self.job = job
+        self.first = first
+        self.stopped = threading.Event()
+        self.wake = threading.Event()
+        self.thread = None
+        self._start()
+        _running.append(self)
+        if hasattr(os, "register_at_fork"):
+            os.register_at_fork(after_in_child=self._start)
+
+    def _start(self):
+        if self.stopped.is_set():
+            return
+        self.wake = threading.Event()
+        self.thread = threading.Thread(target=self._run, daemon=True, name="analytics-platform")
+        self.thread.start()
+
+    def _run(self):
+        wait = self.first
+        while not self.stopped.is_set():
+            self.wake.wait(wait)
+            self.wake.clear()
+            if self.stopped.is_set():
+                return
+            try:
+                wait = self.job()
+            except Exception:  # never let a job die
+                wait = RETRY
+
+    def run_now(self):
+        self.wake.set()
+
+    def stop(self, wait=2.0):
+        """Stops the job and waits (up to wait seconds) for its thread to end."""
+        self.stopped.set()
+        self.wake.set()
+        if self in _running:
+            _running.remove(self)
+        thread = self.thread
+        if wait and thread is not None and thread.is_alive() and thread is not threading.current_thread():
+            thread.join(wait)
+
+
+class Core:
+    def __init__(self, site_id, secret, endpoint, debug=False):
+        self.site_id = str(site_id or "")
+        self.secret = str(secret or "")
+        self.endpoint = str(endpoint or "").rstrip("/")
+        self.valid = bool(re.fullmatch(r"[a-fA-F0-9]{24}", self.site_id) and self.secret and self.endpoint)
+        self.debug = bool(debug)
+        self.user_agent = "analytics-platform-python/" + VERSION
+        self._switches = {}
+        self._headers = []
+        self._flags = {}
+        self._maintenance = {"on": False}
+        self._etag = ""
+        self._loaded = []
+        self._job = None
+
+    def log(self, module, *args):
+        if self.debug:
+            print("[analytics-platform:%s]" % module, *args)
+
+    def send(self, module, path, method="POST", body=None, auth="body", headers=None):
+        """JSON to the collector. auth "body" puts site_id and sitesecret in
+        the body (the event endpoints), "bearer" sends the secret as a header
+        (firewall, switches). Returns a Response, or None when it couldn't be sent."""
+        if not self.valid:
+            return None
+        all_headers = {"User-Agent": self.user_agent}
+        data = None
+        if body is not None:
+            if auth == "body":
+                body = dict(body, site_id=self.site_id, sitesecret=self.secret)
+            data = json.dumps(body).encode("utf-8")
+            all_headers["Content-Type"] = "application/json"
+        if auth == "bearer":
+            all_headers["Authorization"] = "Bearer " + self.secret
+        all_headers.update(headers or {})
+        request = urllib.request.Request(self.endpoint + path, data=data, headers=all_headers, method=method)
+        try:
+            with urllib.request.urlopen(request, timeout=TIMEOUT) as res:
+                return Response(res.status, {k.lower(): v for k, v in res.headers.items()}, res.read())
+        except urllib.error.HTTPError as error:
+            if error.code != 304:
+                self.log(module, "collector answered %s to %s" % (error.code, path))
+            return Response(error.code, {k.lower(): v for k, v in (error.headers or {}).items()}, error.read() if error.fp else b"")
+        except Exception as error:
+            self.log(module, "could not reach the collector (%s): %s" % (path, error))
+            return None
+
+    # ---- Module switches ----
+
+    def is_on(self, name):
+        """False only when the dashboard turned the module off."""
+        return self._switches.get(name) is not False
+
+    def _sync_switches(self):
+        headers = {"X-Platform-Modules": ",".join(self._loaded)}
+        if self._etag:
+            headers["If-None-Match"] = self._etag
+        res = self.send("core", "/events/platform/config?site_id=" + self.site_id, method="GET", auth="bearer", headers=headers)
+        if res is not None and res.status == 200:
+            data = res.json() or {}
+            if data.get("v") == 1 and isinstance(data.get("modules"), dict):
+                self._switches = data["modules"]
+                self._flags = data.get("flags") if isinstance(data.get("flags"), dict) else {}
+                self._maintenance = data.get("maintenance") if isinstance(data.get("maintenance"), dict) else {"on": False}
+                # Only well-formed headers (no line breaks can get through).
+                self._headers = [(p[0], p[1]) for p in data.get("headers") or [] if isinstance(p, list) and len(p) == 2
+                                 and isinstance(p[0], str) and re.fullmatch(r"[A-Za-z][A-Za-z0-9-]{0,63}", p[0])
+                                 and isinstance(p[1], str) and "\r" not in p[1] and "\n" not in p[1] and len(p[1]) <= 4000]
+                self._etag = res.headers.get("etag", "")
+            return max(MIN_SYNC, int(data.get("sync") or RETRY))
+        return RETRY
+
+    def flags(self):
+        return self._flags
+
+    def maintenance(self):
+        return self._maintenance
+
+    def security_headers(self):
+        """The security headers picked on the dashboard, when the module is on."""
+        return list(self._headers) if self.is_on("headers") else []
+
+    def start(self, names):
+        """Starts fetching the switches; names are the modules the code loaded."""
+        self._loaded = list(names)
+        if self.valid:
+            self._job = Periodic(self._sync_switches)
+
+    def stop(self):
+        if self._job:
+            self._job.stop()
+
+
+def header(headers, name):
+    """A header from a dict with lowercase names (or None)."""
+    value = headers.get(name.lower())
+    return value.strip() if isinstance(value, str) else ""

analytics_platform/crawlers.py

@@ -0,0 +1,191 @@
+"""Crawlers module: counts the search engines and AI bots that fetch the site
+and sends one row a minute per bot, page and status to POST /events/crawlers
+(the protocol is sdk/protocols/crawlers.md). This traffic is invisible to the
+browser tracker, because crawlers don't run JavaScript: the server is the only
+place it can be counted.
+
+Only requests whose user agent names a known crawler are counted, so visitors
+are never counted twice. Nothing about a visitor is sent; bot addresses go
+with the batch only so the collector can check a crawler is itself, and are
+not stored.
+"""
+import platform as host
+import re
+import socket
+import threading
+import time
+from datetime import datetime, timezone
+
+from .core import Periodic
+
+FLUSH_EVERY = 60
+MAX_KEYS = 300
+MAX_ROWS = 500
+MAX_IPS_PER_BOT = 5
+MAX_PATH = 400
+
+# The catalogue of the protocol, most specific first: "ChatGPT-User" must be
+# tried before "GPTBot", "Claude-User" before "ClaudeBot". Keep in step with
+# the collector (expresscollect/Analytics/crawlers.js CATALOGUE).
+BOTS = [
+    # Fetched because a person asked an AI about the page.
+    ("ChatGPT-User", r"ChatGPT-User"),
+    ("OAI-SearchBot", r"OAI-SearchBot"),
+    ("Perplexity-User", r"Perplexity-User"),
+    ("Claude-User", r"Claude-User"),
+    ("Claude-SearchBot", r"Claude-SearchBot"),
+    ("MistralAI-User", r"MistralAI-User"),
+    ("MistralAI-SearchBot", r"MistralAI-SearchBot"),
+    ("Meta-ExternalFetcher", r"meta-externalfetcher"),
+    # Bulk collection, for training or a dataset.
+    ("GPTBot", r"GPTBot"),
+    ("ClaudeBot", r"ClaudeBot|Claude-Web"),
+    ("anthropic-ai", r"anthropic-ai"),
+    ("CCBot", r"CCBot"),
+    ("Bytespider", r"Bytespider"),
+    ("Meta-ExternalAgent", r"meta-externalagent|facebookbot"),
+    ("Amazonbot", r"Amazonbot"),
+    ("Diffbot", r"Diffbot"),
+    ("omgili", r"omgili"),
+    ("AI2Bot", r"AI2Bot"),
+    ("Timpibot", r"Timpibot"),
+    ("Webzio-Extended", r"Webzio-Extended"),
+    ("PerplexityBot", r"PerplexityBot"),
+    ("DuckAssistBot", r"DuckAssistBot"),
+    ("YouBot", r"YouBot"),
+    # Search engines. Google's tools before Googlebot itself.
+    ("Google-InspectionTool", r"Google-InspectionTool"),
+    ("Storebot-Google", r"Storebot-Google"),
+    ("AdsBot-Google", r"AdsBot-Google"),
+    ("Googlebot", r"Googlebot|Google-Site-Verification"),
+    ("bingbot", r"bingbot|BingPreview|adidxbot"),
+    ("YandexBot", r"Yandex(Bot|Mobile|Images|Webmaster)?"),
+    ("Baiduspider", r"Baiduspider"),
+    ("DuckDuckBot", r"DuckDuckBot|DuckDuckGo-Favicons"),
+    ("Applebot", r"Applebot"),
+    ("PetalBot", r"PetalBot|AspiegelBot"),
+    ("SeznamBot", r"SeznamBot"),
+    ("Yeti", r"Yeti/"),
+    ("MojeekBot", r"MojeekBot"),
+    ("Qwantbot", r"Qwantbot|Qwantify"),
+    # SEO and backlink tools.
+    ("AhrefsBot", r"AhrefsBot|AhrefsSiteAudit"),
+    ("SemrushBot", r"SemrushBot|SiteAuditBot"),
+    ("DotBot", r"DotBot"),
+    ("MJ12bot", r"MJ12bot"),
+    ("BLEXBot", r"BLEXBot"),
+    ("DataForSeoBot", r"DataForSeoBot"),
+    ("Barkrowler", r"Barkrowler"),
+    ("ScreamingFrog", r"Screaming\s?Frog"),
+    ("ZoominfoBot", r"ZoominfoBot"),
+    # Link previews.
+    ("facebookexternalhit", r"facebookexternalhit"),
+    ("Twitterbot", r"Twitterbot"),
+    ("LinkedInBot", r"LinkedInBot"),
+    ("Slackbot", r"Slackbot|Slack-ImgProxy"),
+    ("Discordbot", r"Discordbot"),
+    ("TelegramBot", r"TelegramBot"),
+    ("WhatsApp", r"WhatsApp"),
+    ("Pinterestbot", r"Pinterest(bot)?/"),
+    ("redditbot", r"redditbot"),
+    # Uptime and speed checks.
+    ("UptimeRobot", r"UptimeRobot"),
+    ("Pingdom", r"Pingdom"),
+    ("BetterUptime", r"Better\s?Uptime|betteruptime"),
+    ("Lighthouse", r"Chrome-Lighthouse|Google Page Speed"),
+    ("GTmetrix", r"GTmetrix"),
+    # Archives.
+    ("ia_archiver", r"ia_archiver"),
+    ("archive.org_bot", r"archive\.org_bot|Wayback"),
+]
+_BOTS = [(name, re.compile(pattern, re.I)) for name, pattern in BOTS]
+
+# Anything else that names itself a bot: counted as "Other bot", so a new
+# crawler shows up as traffic instead of disappearing.
+_GENERIC = re.compile(
+    r"bot\b|crawler|crawling|spider|scrapy|curl/|wget/|python-requests|headlesschrome|http-client|okhttp|java/|go-http-client",
+    re.I,
+)
+
+
+def detect(user_agent):
+    """The crawler's catalogue name, or "" when the user agent isn't one."""
+    ua = str(user_agent or "")
+    if not ua:
+        return ""
+    for name, pattern in _BOTS:
+        if pattern.search(ua):
+            return name
+    return "Other bot" if _GENERIC.search(ua) else ""
+
+
+class Crawlers:
+    """Options: server_name (default the host name)."""
+
+    def __init__(self, core, server_name=None, **_):
+        self.core = core
+        self.server_name = str(server_name or socket.gethostname())[:100]
+        self.runtime = "Python " + host.python_version()
+        self.rows = {}
+        self.agents = {}
+        self.lock = threading.Lock()
+        self.job = Periodic(self._flush_job, FLUSH_EVERY) if core.valid else None
+
+    def record(self, bot, path, status, ms=0, size=0, ip=""):
+        """One finished request from a crawler. bot: detect(user_agent)."""
+        if not bot or not self.core.is_on("crawlers"):
+            return
+        clean = (re.split(r"[?#]", str(path or "/"))[0] or "/")[:MAX_PATH]
+        minute = int(time.time() // 60) * 60
+        with self.lock:
+            key = (minute, bot, clean, int(status or 0))
+            # Past MAX_KEYS a crawler asking for thousands of URLs becomes one row.
+            if key not in self.rows and len(self.rows) >= MAX_KEYS:
+                key = (minute, bot, "(other)", int(status or 0))
+            row = self.rows.get(key)
+            if row is None:
+                row = self.rows[key] = {"count": 0, "bytes": 0, "sum_ms": 0}
+            row["count"] += 1
+            row["bytes"] += max(0, int(size or 0))
+            row["sum_ms"] += max(0, int(ms or 0))
+            if ip:
+                seen = self.agents.setdefault(bot, [])
+                if len(seen) < MAX_IPS_PER_BOT and ip not in seen:
+                    seen.append(ip)
+
+    def middleware(self, user_agent, path, status, ms=0, size=0, ip=""):
+        """Records a finished request when it came from a crawler."""
+        self.record(detect(user_agent), path, status, ms, size, ip)
+
+    def flush(self, everything=True):
+        current = int(time.time() // 60) * 60
+        with self.lock:
+            ready = [(k, v) for k, v in self.rows.items() if everything or k[0] < current]
+            for k, _ in ready:
+                del self.rows[k]
+            sent = {k[1] for k, _ in ready}
+            agents = [{"bot": bot, "ips": list(ips)} for bot, ips in self.agents.items() if bot in sent]
+            if ready:
+                # A bot with rows still waiting (this minute isn't over) keeps
+                # its addresses, so the next batch can be checked as well.
+                waiting = {key[1] for key in self.rows}
+                self.agents = {bot: ips for bot, ips in self.agents.items() if bot in waiting}
+        rows = [{
+            "minute": datetime.fromtimestamp(k[0], timezone.utc).isoformat().replace("+00:00", "Z"),
+            "bot": k[1], "path": k[2], "status": k[3],
+            "count": v["count"], "bytes": v["bytes"], "sum_ms": v["sum_ms"],
+        } for k, v in ready]
+        for i in range(0, len(rows), MAX_ROWS):
+            self.core.send("crawlers", "/events/crawlers", body={
+                "server_name": self.server_name, "runtime": self.runtime,
+                "rows": rows[i:i + MAX_ROWS], "agents": agents if i == 0 else [],
+            })
+
+    def _flush_job(self):
+        self.flush(everything=False)
+        return FLUSH_EVERY
+
+    def close(self):
+        if self.job:
+            self.job.stop()
+        self.flush()

analytics_platform/django.py

@@ -0,0 +1,112 @@
+"""Django: add the middleware first in MIDDLEWARE and the options to settings.
+
+    MIDDLEWARE = ["analytics_platform.django.PlatformMiddleware", ...]
+    ANALYTICS_PLATFORM = {
+        "site_id": "...",
+        "secret": os.environ["ANALYTICS_SITE_SECRET"],
+        "endpoint": "https://...",
+    }
+
+In your code, get_platform() is the same platform: get_platform().track(...).
+The visitor's address is REMOTE_ADDR unless you set trust_proxy.
+"""
+import time
+
+from django.conf import settings
+from django.http import HttpResponse
+
+from . import SESSION_HEADER, Platform
+
+_platform = None
+
+
+def get_platform():
+    global _platform
+    if _platform is None:
+        _platform = Platform(**getattr(settings, "ANALYTICS_PLATFORM", {}))
+        _connect_login_failed()
+    return _platform
+
+
+def _connect_login_failed():
+    """Failed Django logins count for bot protection, on their own."""
+    try:
+        from django.contrib.auth.signals import user_login_failed
+    except Exception:
+        return
+
+    def on_failed(sender, credentials=None, request=None, **_):
+        if request is not None:
+            get_platform().login_failed(request)
+
+    user_login_failed.connect(on_failed, weak=False, dispatch_uid="analytics_platform_login_failed")
+
+    # Logins and logouts in the audit log.
+    from django.contrib.auth.signals import user_logged_in, user_logged_out
+
+    def on_login(sender, request=None, user=None, **_):
+        get_platform().audit("user.login", actor=getattr(user, "get_username", lambda: "")() if user else "", request=request)
+
+    def on_logout(sender, request=None, user=None, **_):
+        get_platform().audit("user.logout", actor=getattr(user, "get_username", lambda: "")() if user else "", request=request)
+
+    user_logged_in.connect(on_login, weak=False, dispatch_uid="analytics_platform_login")
+    user_logged_out.connect(on_logout, weak=False, dispatch_uid="analytics_platform_logout")
+
+
+class PlatformMiddleware:
+    def __init__(self, get_response):
+        self.get_response = get_response
+        self.platform = get_platform()
+
+    def __call__(self, request):
+        headers = {k.lower(): v for k, v in request.headers.items()}
+        started = time.perf_counter()
+        ip = request.META.get("REMOTE_ADDR", "")
+        # The tracker's own addresses, answered before anything else: a
+        # visitor's beacon is our traffic, not a page of the site.
+        answered = self.platform.proxy_response(
+            request.path, request.method, request.META.get("QUERY_STRING", ""),
+            headers, request.body if request.method == "POST" else None, ip)
+        if answered:
+            status, extra, body = answered
+            response = HttpResponse(body, status=status)
+            for name, value in extra:
+                response[name] = value
+            return response
+        blocked = self.platform.blocked(ip, headers, request.method, request.path)
+        if blocked:
+            status, body, extra = blocked
+            response = HttpResponse(body, status=status)
+            for name, value in extra:
+                response[name] = value
+            # A crawler the firewall turned away is still a crawler, and
+            # counted with the status it got.
+            self.platform.record_crawler(headers.get("user-agent", ""), request.path, status, started, 0, ip)
+            return response
+        closed = self.platform.maintenance_response(request.META.get("REMOTE_ADDR", ""), headers, request.path, request.META.get("QUERY_STRING", ""))
+        if closed and closed[0] != "pass":
+            response = HttpResponse(closed[2], status=503)
+            for name, value in closed[1]:
+                response[name] = value
+            return response
+        response = self.get_response(request)
+        for name, value in self.platform.security_headers(response.headers.keys()):
+            response[name] = value
+        if closed:
+            response["Set-Cookie"] = closed[1][0][1]
+        match = getattr(request, "resolver_match", None)
+        route = getattr(match, "route", "") or ""
+        self.platform.time_request(request.method, ("/" + route.lstrip("^").rstrip("$").lstrip("/")) if route else "",
+                                   response.status_code, started, request.path)
+        self.platform.record_crawler(headers.get("user-agent", ""), request.path, response.status_code, started,
+                                     response.headers.get("Content-Length") or 0, ip)
+        return response
+
+    def process_exception(self, request, exception):
+        match = getattr(request, "resolver_match", None)
+        self.platform.capture_exception(exception, status=500, kind="error", request={
+            "method": request.method, "path": request.path, "route": getattr(match, "route", "") or "",
+            "session_id": request.headers.get(SESSION_HEADER, ""),
+        })
+        return None

analytics_platform/errors.py

@@ -0,0 +1,113 @@
+"""Error monitoring module: the server's errors go to the collector (POST
+/events/server-errors), where they join the browser's errors in the Errors
+report. Uncaught exceptions (main thread and other threads) are captured on
+their own and Python still prints and exits as it would. The framework
+integrations capture request errors (5xx); capture_exception() captures
+errors you catch. Errors are sent in small batches; sending never raises.
+"""
+import os
+import platform as host
+import re
+import socket
+import sys
+import threading
+import traceback
+
+from .core import Periodic
+
+BATCH = 10
+QUEUE_MAX = 100
+FLUSH_EVERY = 2
+
+
+class Errors:
+    """Options: server_name (default the host name), release (which build this
+    is; default the ANALYTICS_RELEASE environment variable), root (paths in tracebacks
+    are made relative to it; default the working directory), capture_uncaught
+    (default True), before_send(event, error) to change an event or drop it
+    (return None)."""
+
+    def __init__(self, core, server_name=None, root=None, release=None, capture_uncaught=True, before_send=None, **_):
+        self.core = core
+        self.server_name = str(server_name or socket.gethostname())[:100]
+        self.root = str(root or os.getcwd()).rstrip("/\\")
+        # Which build this is, for "introduced in" in the Errors report.
+        self.release = re.sub(r"[^\w.+\-]", "", str(release or os.environ.get("ANALYTICS_RELEASE", "")).strip())[:64]
+        self.before_send = before_send
+        self.queue = []
+        self.lock = threading.Lock()
+        self.job = Periodic(self._flush_job, FLUSH_EVERY) if core.valid else None
+        self._previous = None
+        if capture_uncaught:
+            self._previous = (sys.excepthook, threading.excepthook)
+            sys.excepthook = self._excepthook
+            threading.excepthook = self._thread_excepthook
+
+    def _excepthook(self, kind, value, tb):
+        self.capture_exception(value, kind="error")
+        self.flush()
+        self._previous[0](kind, value, tb)
+
+    def _thread_excepthook(self, args):
+        if args.exc_value is not None:
+            self.capture_exception(args.exc_value, kind="error")
+            self.flush()
+        self._previous[1](args)
+
+    def _relative(self, stack):
+        return stack.replace(self.root + os.sep, "").replace(self.root + "/", "") if self.root else stack
+
+    def capture_exception(self, error, request=None, status=0, kind="handled"):
+        """request: {"method", "path", "route", "session_id"} (the integrations fill it in)."""
+        if not self.core.is_on("errors"):
+            return
+        try:
+            is_error = isinstance(error, BaseException)
+            stack = "".join(traceback.format_exception(type(error), error, error.__traceback__)) if is_error else ""
+            event = {
+                "message": str(error) or (type(error).__name__ if is_error else "Unknown error"),
+                "error_type": type(error).__name__ if is_error else "Error",
+                "stack": self._relative(stack),
+                "kind": kind,
+                "status_code": int(status or 0),
+                "runtime": "Python",
+                "runtime_version": host.python_version(),
+                "os": sys.platform,
+                "server_name": self.server_name,
+                "release": self.release,
+            }
+            event.update({k: str(v or "") for k, v in (request or {}).items() if k in ("method", "path", "route", "session_id")})
+            if event.get("path"):
+                event["path"] = event["path"].split("?")[0]
+            if self.before_send:
+                event = self.before_send(event, error)
+                if not event:
+                    return
+            with self.lock:
+                if len(self.queue) >= QUEUE_MAX:
+                    self.core.log("errors", "queue full, error dropped")
+                    return
+                self.queue.append(event)
+                full = len(self.queue) >= BATCH
+            if full and self.job:
+                self.job.run_now()
+        except Exception as failure:
+            self.core.log("errors", "could not capture error:", failure)
+
+    def flush(self):
+        with self.lock:
+            waiting, self.queue = self.queue, []
+        for i in range(0, len(waiting), BATCH):
+            self.core.send("errors", "/events/server-errors", body={"errors": waiting[i:i + BATCH]})
+
+    def _flush_job(self):
+        self.flush()
+        return FLUSH_EVERY
+
+    def close(self):
+        if self.job:
+            self.job.stop()
+        if self._previous:
+            sys.excepthook, threading.excepthook = self._previous
+            self._previous = None
+        self.flush()