analytics-platform 0.2.1__tar.gz → 0.2.2__tar.gz

{analytics_platform-0.2.1 → analytics_platform-0.2.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: analytics-platform
-Version: 0.2.1
+Version: 0.2.2
 Summary: Firewall, error monitoring and server analytics for your Python server, run from your analytics dashboard.
 Author: steviec87
 License-Expression: MIT
@@ -103,7 +103,7 @@ create_platform(site_id=..., secret=..., endpoint=...,
                 analytics=False)
 ```
 
-**Switch modules off in the dashboard** (Security → Settings → Server package) without a
+**Switch modules off in the dashboard** (Security → Server package) without a
 deploy: the package checks about once a minute. Modules are independent: one
 failing to start, or the collector being down, never stops the others, and
 never slows your requests. Background work runs in daemon threads, which start

{analytics_platform-0.2.1 → analytics_platform-0.2.2}/README.md

@@ -77,7 +77,7 @@ create_platform(site_id=..., secret=..., endpoint=...,
                 analytics=False)
 ```
 
-**Switch modules off in the dashboard** (Security → Settings → Server package) without a
+**Switch modules off in the dashboard** (Security → Server package) without a
 deploy: the package checks about once a minute. Modules are independent: one
 failing to start, or the collector being down, never stops the others, and
 never slows your requests. Background work runs in daemon threads, which start

{analytics_platform-0.2.1 → analytics_platform-0.2.2}/analytics_platform/__init__.py

@@ -16,7 +16,7 @@ No dependencies; Python 3.8+.
 import asyncio
 import time
 
-from .core import Core, SESSION_HEADER, VERSION, header
+from .core import Core, SESSION_HEADER, VERSION, header, request_target
 
 __all__ = ["create_platform", "Platform", "PlatformMiddleware", "VERSION"]
 
@@ -286,7 +286,8 @@ class Platform:
 
     def blocked(self, remote, headers, method, path):
         """(status, body, headers) when the firewall turns the request away,
-        else None. headers: a dict with lowercase names."""
+        else None. headers: a dict with lowercase names. path: the request
+        target as sent, query string and all, not yet decoded (request_target)."""
         firewall = self._modules.get("firewall")
         if not firewall:
             return None
@@ -321,7 +322,7 @@ class Platform:
                 start_response("%d " % status, list(extra))
                 return [body]
             blocked = platform.blocked(environ.get("REMOTE_ADDR", ""), headers, environ.get("REQUEST_METHOD", ""),
-                                       environ.get("PATH_INFO", ""))
+                                       _wsgi_target(environ))
             if blocked:
                 status, body, extra = blocked
                 start_response("%d %s" % (status, body.decode()), extra)
@@ -387,6 +388,17 @@ class Platform:
         return PlatformMiddleware(app, platform=self)
 
 
+def _wsgi_target(environ):
+    # gunicorn keeps the target as sent (RAW_URI), uWSGI and mod_wsgi too
+    # (REQUEST_URI); PATH_INFO is decoded, and PEP 3333 hands its bytes over as latin-1.
+    path = environ.get("SCRIPT_NAME", "") + environ.get("PATH_INFO", "")
+    try:
+        path = path.encode("latin-1")
+    except UnicodeEncodeError:
+        pass
+    return request_target(path, environ.get("QUERY_STRING", ""), environ.get("RAW_URI") or environ.get("REQUEST_URI"))
+
+
 def _wsgi_body(environ):
     try:
         length = int(environ.get("CONTENT_LENGTH") or 0)
@@ -439,7 +451,9 @@ class PlatformMiddleware:
                         "headers": [(k.lower().encode(), str(v).encode()) for k, v in extra]})
             await send({"type": "http.response.body", "body": out})
             return
-        blocked = self.platform.blocked(client[0] or "", headers, scope.get("method", ""), scope.get("path", ""))
+        # raw_path is the path as sent; servers that leave it out get path encoded again.
+        target = request_target(scope.get("path", ""), scope.get("query_string", b""), scope.get("raw_path"))
+        blocked = self.platform.blocked(client[0] or "", headers, scope.get("method", ""), target)
         if blocked:
             status, body, extra = blocked
             await send({"type": "http.response.start", "status": status,

{analytics_platform-0.2.1 → analytics_platform-0.2.2}/analytics_platform/core.py

@@ -13,9 +13,10 @@ import os
 import re
 import threading
 import urllib.error
+import urllib.parse
 import urllib.request
 
-VERSION = "0.1.0"
+VERSION = "0.2.2"
 TIMEOUT = 5
 MIN_SYNC = 30
 RETRY = 60
@@ -197,3 +198,27 @@ def header(headers, name):
     """A header from a dict with lowercase names (or None)."""
     value = headers.get(name.lower())
     return value.strip() if isinstance(value, str) else ""
+
+
+# The characters a path may carry unencoded (RFC 3986), which quote() leaves alone.
+_PATH_SAFE = "/:@!$&'()*+,;=-._~"
+
+
+def request_target(path, query="", raw=None):
+    """The request target as the visitor sent it, query string and all, which
+    is what the request rules match (sdk/protocols/firewall.md): the Node and
+    PHP clients hand over req.url and REQUEST_URI. raw: the server's own copy
+    of it when it keeps one. Otherwise the path the framework has already
+    decoded is encoded again, so the rules' one decoding pass is the only one.
+    path: text, or the bytes it was decoded from."""
+    if isinstance(raw, bytes):
+        raw = raw.decode("latin-1")
+    if isinstance(raw, str) and raw:
+        target = raw
+    else:
+        target = urllib.parse.quote(path or "", safe=_PATH_SAFE)
+    if isinstance(query, bytes):
+        query = query.decode("latin-1")
+    if query and "?" not in target:
+        target += "?" + query
+    return target

{analytics_platform-0.2.1 → analytics_platform-0.2.2}/analytics_platform/django.py

@@ -74,7 +74,9 @@ class PlatformMiddleware:
             for name, value in extra:
                 response[name] = value
             return response
-        blocked = self.platform.blocked(ip, headers, request.method, request.path)
+        # The target as sent: request.path is decoded, get_full_path() encodes it again.
+        target = request.META.get("RAW_URI") or request.META.get("REQUEST_URI") or request.get_full_path()
+        blocked = self.platform.blocked(ip, headers, request.method, target)
         if blocked:
             status, body, extra = blocked
             response = HttpResponse(body, status=status)

{analytics_platform-0.2.1 → analytics_platform-0.2.2}/analytics_platform/firewall.py

@@ -292,8 +292,9 @@ class Firewall:
                     self._ban_here(str(ip), matched["ban"])
         if not rule and rules["rate"] and not rules["goodbots"].has(key) and self._over_rate(str(ip), rules["rate"]):
             rule = "rate"
-            # Banned here straight away; the report bans it on the site's other servers.
-            self._ban_here(str(ip), rules["ban"])
+            # Banned here straight away in block mode, as a probe is; the report bans it on the other servers.
+            if action == "block":
+                self._ban_here(str(ip), rules["ban"])
         if not rule:
             return allow
         self._report({

{analytics_platform-0.2.1 → analytics_platform-0.2.2}/analytics_platform/proxy.py

@@ -27,6 +27,7 @@ import urllib.request
 FILES = frozenset((
     "track.js", "s.js", "snapshot.js", "replay.js", "survey.js", "banner.js",
     "leadform.js", "notice.js", "editor.js", "picker.js", "feedback.js", "shield.js",
+    "widget.js", "share.js",
 ))
 # A pinned version never changes; /v<major>/ follows new releases, so it is
 # held for an hour, the same as the collector serves it with.

{analytics_platform-0.2.1 → analytics_platform-0.2.2}/analytics_platform/request_rules.py

@@ -29,14 +29,18 @@ def safe_pattern(pattern):
     """A pattern can only come from us, but a careless one would be matched
     against text an attacker chooses, in this very request, and `re` can't be
     timed out. So the client checks the shape itself instead of trusting what it
-    was handed: a quantifier on a group that itself repeats ("(a+)+") takes
-    exponential time on input that nearly matches. Backreferences and lookbehind
-    are refused for the same reason."""
+    was handed. A repeated group that can match the same text more than one way
+    takes exponential time on input that nearly matches; a group can do that
+    when it holds a quantifier ("(a+)+", "(\\w\\d?)+") or an alternation
+    ("(a|ab)*", "(\\w|\\d)+"), so a group followed by * + or { holds neither.
+    Backreferences and lookbehind are refused too. The same check as
+    backend/services/firewall/requestRules.js, free repeats included."""
     if not isinstance(pattern, str) or not pattern or len(pattern) > MAX_REGEX:
         return False
     if re.search(r"\\[1-9]", pattern) or "(?<" in pattern:
         return False
-    open_at = []
+    # Per open group: whether anything in it, nested groups too, repeats or branches.
+    open_groups = []
     i = 0
     while i < len(pattern):
         char = pattern[i]
@@ -44,25 +48,97 @@ def safe_pattern(pattern):
             i += 2
             continue
         if char == "[":
-            # A character class: quantifiers inside it are literals.
-            i += 1
+            # A character class: skipped whole, quantifiers inside it are literals.
             while i < len(pattern) and pattern[i] != "]":
                 i += 2 if pattern[i] == "\\" else 1
             i += 1
             continue
         if char == "(":
-            open_at.append(i)
+            open_groups.append(False)
+            # "(?:", "(?=", "(?!": that ? names the kind of group, it repeats nothing.
+            if pattern[i + 1:i + 2] == "?":
+                i += 2
         elif char == ")":
-            if not open_at:
+            if not open_groups:
+                return False
+            ambiguous = open_groups.pop()
+            nxt = pattern[i + 1:i + 2]
+            if ambiguous and nxt in ("*", "+", "{"):
                 return False
-            start = open_at.pop()
-            nxt = pattern[i + 1] if i + 1 < len(pattern) else ""
-            if nxt in ("*", "+", "{"):
-                inside = pattern[start + 1:i]
-                if re.search(r"(^|[^\\])[*+]", inside) or re.search(r"(^|[^\\])\{[0-9]", inside):
-                    return False
+            if ambiguous and open_groups:
+                open_groups[-1] = True
+        elif open_groups and char in "|*+?{":
+            open_groups[-1] = True
+        i += 1
+    return not open_groups and _free_repeats(pattern) <= 1
+
+
+def _literal_at(pattern, i):
+    if i >= len(pattern):
+        return None
+    char = pattern[i]
+    if char == "\\":
+        following = pattern[i + 1:i + 2]
+        return following if following and not re.match(r"[a-z0-9]", following, re.I) else None
+    return None if char in ".[](){}*+?|^$" else char
+
+
+def _free_repeats(pattern):
+    """Two repeats that can both take the same text multiply each other's
+    work: ".*=.*=.*x" against a query of "=" that never ends in "x" is
+    polynomial, and 2,000 characters is plenty for that to hang a request. A
+    repeat is pinned when the character straight after it is one it can't
+    match ("[^/]+/" ends at the first "/", once); any other is free, and a
+    pattern may hold one free repeat."""
+    free = 0
+    i = 0
+    while i < len(pattern):
+        start = i
+        char = pattern[i]
+        if char == "\\":
+            i += 2
+        elif char == "[":
+            i += 1
+            while i < len(pattern) and pattern[i] != "]":
+                i += 2 if pattern[i] == "\\" else 1
+            i += 1
+        elif char == "(":
+            i += 3 if pattern[i + 1:i + 2] == "?" else 1
+            continue
+        elif char in "|^$":
+            i += 1
+            continue
+        else:
+            i += 1
+        # A group's text can't be tried against one character.
+        atom = None if char == ")" else pattern[start:i]
+        quantifier = pattern[i:i + 1]
+        repeats = quantifier in ("*", "+")
+        if quantifier == "{":
+            bounds = re.match(r"\{(\d*)(,(\d*))?\}", pattern[i:])
+            if not bounds:
+                continue
+            # "{3}" is one length, not a choice; "{2,}" and "{2,9}" are choices.
+            repeats = bounds.group(2) is not None and bounds.group(3) != bounds.group(1)
+            i += len(bounds.group(0)) - 1
+        elif quantifier != "?" and not repeats:
+            continue
         i += 1
-    return not open_at
+        if pattern[i:i + 1] == "?":
+            i += 1
+        if not repeats:
+            continue
+        nxt = _literal_at(pattern, i)
+        pinned = False
+        if atom and nxt is not None:
+            try:
+                pinned = re.fullmatch("(?:%s)" % atom, nxt, re.IGNORECASE) is None
+            except re.error:
+                # Not an atom on its own: free.
+                pass
+        if not pinned:
+            free += 1
+    return free
 
 
 def _section(section):
@@ -157,10 +233,13 @@ def match_request(rules, method="", path="", user_agent=""):
     target = str(path or "")
     mark = target.find("?")
     raw_path = _normalize_path((target if mark == -1 else target[:mark])[:MAX_TEXT])
-    raw_query = ("" if mark == -1 else target[mark + 1:])[:MAX_TEXT].lower()
+    query = ("" if mark == -1 else target[mark + 1:])[:MAX_TEXT]
+    raw_query = query.lower()
     ua = str(user_agent or "")[:MAX_TEXT].lower()
     decoded_path = _normalize_path(_decode_once((target if mark == -1 else target[:mark])[:MAX_TEXT])[:MAX_TEXT])
-    decoded_query = _decode_once(raw_query)[:MAX_TEXT]
+    # Lower-cased after decoding: %55NION decodes to "UNION", which a
+    # lower-cased "union select" rule would otherwise never see.
+    decoded_query = _decode_once(query)[:MAX_TEXT].lower()
     upper = str(method or "").upper()
     for rule in rules:
         if rule["methods"] and upper not in rule["methods"]:

{analytics_platform-0.2.1 → analytics_platform-0.2.2}/analytics_platform/resources.py

@@ -87,9 +87,16 @@ def cpu_fields(text):
     return None
 
 
+def cpu_total(fields):
+    """Every jiffy in the counters once. guest and guest_nice (columns 9 and
+    10) are already inside user and nice, so they are left out, as psutil and
+    top do: counting them again reads a busy hypervisor as half idle."""
+    return sum(fields[:8])
+
+
 def cpu_times_of(fields):
     """(busy, total) jiffies from the counters, for cpu_percent."""
-    total = sum(fields)
+    total = cpu_total(fields)
     idle = fields[3] + (fields[4] if len(fields) > 4 else 0)
     return total - idle, total
 
@@ -122,7 +129,7 @@ def cpu_shares(before, after):
     will change."""
     if not before or not after:
         return {}
-    total = sum(after) - sum(before)
+    total = cpu_total(after) - cpu_total(before)
     if total <= 0:
         return {}
     out = {}
@@ -410,6 +417,11 @@ class Resources:
         if uptime is not None:
             row["host_uptime_s"] = uptime
         with self.lock:
+            if not self.enabled:
+                # Off in the dashboard: only the newest minute is kept, as the
+                # one row that asks the collector, now and then, whether it is
+                # back on (the answer is the only place that says so).
+                self.rows.clear()
             self.rows[row["minute"]] = row
             # Oldest first out: a long outage keeps the recent minutes.
             while len(self.rows) > MAX_ROWS:
@@ -462,10 +474,17 @@ class Resources:
             with self.lock:
                 self.rows.clear()
             return max(OFF_EVERY, self.every)
-        return self.every if self.flush() else RETRY_EVERY
+        sent = self.flush()
+        if not self.enabled:
+            # The collector said the module is off: ask again later, as the agent does.
+            return max(OFF_EVERY, self.every)
+        return self.every if sent else RETRY_EVERY
 
     def close(self):
         if self.sampler:
             self.sampler.stop()
         if self.job:
             self.job.stop()
+        # What's buffered goes out, as with every other module.
+        if self.core.is_on("resources") and self.enabled:
+            self.flush()

{analytics_platform-0.2.1 → analytics_platform-0.2.2}/analytics_platform.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: analytics-platform
-Version: 0.2.1
+Version: 0.2.2
 Summary: Firewall, error monitoring and server analytics for your Python server, run from your analytics dashboard.
 Author: steviec87
 License-Expression: MIT
@@ -103,7 +103,7 @@ create_platform(site_id=..., secret=..., endpoint=...,
                 analytics=False)
 ```
 
-**Switch modules off in the dashboard** (Security → Settings → Server package) without a
+**Switch modules off in the dashboard** (Security → Server package) without a
 deploy: the package checks about once a minute. Modules are independent: one
 failing to start, or the collector being down, never stops the others, and
 never slows your requests. Background work runs in daemon threads, which start

{analytics_platform-0.2.1 → analytics_platform-0.2.2}/analytics_platform.egg-info/SOURCES.txt

@@ -23,6 +23,7 @@ analytics_platform.egg-info/SOURCES.txt
 analytics_platform.egg-info/dependency_links.txt
 analytics_platform.egg-info/top_level.txt
 tests/test_crawlers.py
+tests/test_firewall.py
 tests/test_proxy.py
 tests/test_request_rules.py
 tests/test_resources.py

{analytics_platform-0.2.1 → analytics_platform-0.2.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "analytics-platform"
-version = "0.2.1"
+version = "0.2.2"
 description = "Firewall, error monitoring and server analytics for your Python server, run from your analytics dashboard."
 readme = "README.md"
 requires-python = ">=3.8"

analytics_platform-0.2.2/tests/test_firewall.py

@@ -0,0 +1,56 @@
+"""The firewall's own decisions: when an address gets banned here, and what
+the middleware hands the request rules.
+
+    python3 -m unittest discover -s tests
+"""
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from analytics_platform import _wsgi_target  # noqa: E402
+from analytics_platform.firewall import Firewall  # noqa: E402
+
+
+class FakeCore:
+    valid = False
+    site_id = "a" * 24
+
+    def log(self, *_args):
+        pass
+
+
+def firewall(mode):
+    wall = Firewall(FakeCore())
+    wall.rules = wall._compile({"v": 1, "enabled": True, "mode": mode, "rate": {"limit": 2, "window": 60},
+                                "ban": {"seconds": 600}})
+    return wall
+
+
+class RateBan(unittest.TestCase):
+    def test_monitor_mode_bans_nobody(self):
+        wall = firewall("monitor")
+        rules = [wall.check("203.0.113.9", "GET", "/")["rule"] for _ in range(5)]
+        self.assertEqual(rules, ["", "", "rate", "rate", "rate"], "over the limit, reported as rate, never banned")
+        self.assertEqual(wall.bans, {})
+
+    def test_block_mode_bans_here(self):
+        wall = firewall("block")
+        rules = [wall.check("203.0.113.9", "GET", "/")["rule"] for _ in range(4)]
+        self.assertEqual(rules, ["", "", "rate", "ban"])
+
+
+class WsgiTarget(unittest.TestCase):
+    def test_the_raw_target_is_used_when_the_server_keeps_it(self):
+        self.assertEqual(_wsgi_target({"RAW_URI": "/a%252e?q=1", "PATH_INFO": "/a%2e", "QUERY_STRING": "q=1"}),
+                         "/a%252e?q=1")
+
+    def test_otherwise_path_info_is_encoded_again_with_its_query(self):
+        # PEP 3333: the bytes as latin-1; here a UTF-8 "é".
+        environ = {"SCRIPT_NAME": "/app", "PATH_INFO": "/café", "QUERY_STRING": "id=1%20union"}
+        self.assertEqual(_wsgi_target(environ), "/app/caf%C3%A9?id=1%20union")
+
+
+if __name__ == "__main__":
+    unittest.main()

{analytics_platform-0.2.1 → analytics_platform-0.2.2}/tests/test_request_rules.py

@@ -63,6 +63,14 @@ class PatternSafety(unittest.TestCase):
         for pattern in (r"/(wp-content/uploads|uploads)/[^?]*\.(php[0-9]?|phtml)$", r"(includes|inc|lib)?/?[^/]*\.php$"):
             self.assertTrue(safe_pattern(pattern), pattern)
 
+    def test_the_shared_pattern_lists_decide_the_same_way(self):
+        with open(CASES) as handle:
+            patterns = json.load(handle)["patterns"]
+        for pattern in patterns["refused"]:
+            self.assertFalse(safe_pattern(pattern), pattern)
+        for pattern in patterns["allowed"]:
+            self.assertTrue(safe_pattern(pattern), pattern)
+
     def test_an_unsafe_rule_is_dropped_and_matching_stays_quick(self):
         rules = compile_rules({"rules": [
             {"id": "slow", "action": "block", "path": {"regex": "(a+)+b"}},
@@ -74,5 +82,29 @@ class PatternSafety(unittest.TestCase):
         self.assertLess(time.time() - started, 1.0)
 
 
+class RequestTarget(unittest.TestCase):
+    """The rules match the target as sent; frameworks hand over a decoded path."""
+
+    def test_a_decoded_path_is_encoded_again_so_it_is_decoded_once(self):
+        from analytics_platform.core import request_target
+        # Sent as /a%252e%252e/, which the framework decoded to /a%2e%2e/.
+        self.assertEqual(request_target("/a%2e%2e/"), "/a%252e%252e/")
+        rules = compile_rules({"rules": [{"id": "dots", "action": "block", "path": {"contains": ["/a../"]}}]})
+        self.assertIsNone(match_request(rules, "GET", request_target("/a%2e%2e/"), ""))
+
+    def test_the_query_string_reaches_the_rules(self):
+        from analytics_platform.core import request_target
+        rules = compile_rules({"rules": [{"id": "sqli", "action": "block", "query": {"contains": ["union select"]}}]})
+        target = request_target("/item", "id=1%20union%20select")
+        self.assertEqual(target, "/item?id=1%20union%20select")
+        self.assertEqual(match_request(rules, "GET", target, "")["id"], "sqli")
+
+    def test_the_servers_own_copy_wins(self):
+        from analytics_platform.core import request_target
+        self.assertEqual(request_target("/x y", "a=1", b"/x%20y?a=1"), "/x%20y?a=1")
+        self.assertEqual(request_target("/x y", "a=1", b"/x%20y"), "/x%20y?a=1")
+        self.assertEqual(request_target("/caf\u00e9"), "/caf%C3%A9")
+
+
 if __name__ == "__main__":
     unittest.main()

{analytics_platform-0.2.1 → analytics_platform-0.2.2}/tests/test_resources.py

@@ -297,3 +297,38 @@ class Detail(unittest.TestCase):
         row = mod.disk_rows(["/"], statvfs=lambda _m: Stat())[0]
         self.assertEqual(row["inodes_total"], 262_144)
         self.assertEqual(row["inodes_used"], 62_144)
+
+
+class ReviewFixes(unittest.TestCase):
+    def test_guest_time_is_not_counted_twice(self):
+        # guest (column 9) is already inside user: 400 of the 800 busy jiffies.
+        before = [1000, 0, 0, 8000, 0, 0, 0, 0, 0, 0]
+        after = [1800, 0, 0, 8200, 0, 0, 0, 0, 400, 0]
+        self.assertEqual(mod.cpu_percent(mod.cpu_times_of(before), mod.cpu_times_of(after)), 80.0)
+        self.assertEqual(mod.cpu_shares(before, after)["cpu_user_pct"], 80.0)
+
+    def test_the_collectors_off_answer_slows_reporting_and_keeps_one_row(self):
+        core = FakeCore(Answer(status=200, body={"v": 1, "enabled": False, "every": 300}))
+        checks = Resources(core)
+        checks.rows = {"2026-10-20T12:00:00Z": {"minute": "2026-10-20T12:00:00Z"}}
+        self.assertGreaterEqual(checks._report(), mod.OFF_EVERY)
+        self.assertFalse(checks.enabled)
+        checks.last_at -= 120
+        checks.sample()
+        checks.last_at -= 120
+        checks.sample()
+        self.assertEqual(len(checks.rows), 1, "only the row that asks again")
+        core.answer = Answer()
+        self.assertEqual(checks._report(), checks.every)
+        self.assertTrue(checks.enabled)
+
+    def test_close_sends_what_is_buffered(self):
+        core = FakeCore()
+        checks = Resources(core)
+        checks.rows = {"2026-10-20T12:00:00Z": {"minute": "2026-10-20T12:00:00Z"}}
+        checks.close()
+        self.assertEqual(len(core.sent), 1)
+        core.on = False
+        checks.rows = {"2026-10-20T12:01:00Z": {"minute": "2026-10-20T12:01:00Z"}}
+        checks.close()
+        self.assertEqual(len(core.sent), 1, "nothing goes out once the module is switched off")