analytics-platform 0.3.9__tar.gz → 0.3.11__tar.gz

{analytics_platform-0.3.9 → analytics_platform-0.3.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: analytics-platform
-Version: 0.3.9
+Version: 0.3.11
 Summary: Firewall, error monitoring and server analytics for your Python server, run from your analytics dashboard.
 Author: steviec87
 License-Expression: MIT

{analytics_platform-0.3.9 → analytics_platform-0.3.11}/analytics_platform/__init__.py

@@ -453,7 +453,8 @@ class Platform:
         try:
             ip = firewall.client_ip(remote, headers)
             decision = firewall.check(ip, method, path, header(headers, "user-agent"),
-                                      header(headers, "accept"), header(headers, "cookie"))
+                                      header(headers, "accept"), header(headers, "cookie"),
+                                      fetch_site=header(headers, "sec-fetch-site"))
             if decision["action"] == "block" and decision["rule"] == "challenge":
                 status, extra, body = firewall.challenge_page(ip, method)
                 return status, body, extra

{analytics_platform-0.3.9 → analytics_platform-0.3.11}/analytics_platform/core.py

@@ -23,7 +23,7 @@ import urllib.error
 import urllib.parse
 import urllib.request
 
-VERSION = "0.3.9"
+VERSION = "0.3.11"
 TIMEOUT = 5
 MIN_SYNC = 30
 RETRY = 60
@@ -447,6 +447,24 @@ def header(headers, name):
 _PATH_SAFE = "/:@!$&'()*+,;=-._~"
 
 
+# A scheme, then the authority only when "//" follows it (firewall.md, "Matching").
+_ABSOLUTE = re.compile(r"^[a-zA-Z][a-zA-Z0-9+.-]*:(?://[^/?#]*)?")
+
+
+def origin_form(target):
+    """The request target as the app routes it: an absolute-form one ("GET
+    http://host/admin", sent as if to a proxy) is routed by its path, so it is
+    cut to "/admin"; matched as written it began "http:" and missed every rule
+    on "/admin". gunicorn's urlsplit reads "x:/admin" and "x:admin" as
+    "/admin" too, with no "//" at all, so the scheme goes whatever follows it.
+    "*" and a target already "/..." are kept as they are."""
+    text = str(target or "")
+    if not text or text.startswith("/") or text == "*":
+        return text
+    rest = _ABSOLUTE.sub("", text, count=1)
+    return rest if rest.startswith("/") else "/" + rest
+
+
 def request_target(path, query="", raw=None):
     """The request target as the visitor sent it, query string and all, which
     is what the request rules match (sdk/protocols/firewall.md): the Node and

{analytics_platform-0.3.9 → analytics_platform-0.3.11}/analytics_platform/firewall.py

@@ -19,7 +19,7 @@ import time
 from datetime import datetime, timezone
 
 from . import challenge as browser_check
-from .core import Periodic, RETRY, header
+from .core import Periodic, RETRY, header, origin_form
 from .request_rules import MAX_TEXT, _decode_once, compile_limits, compile_rules, match_limits, match_request, normalize_target, prepare
 
 PROTOCOL = 1
@@ -139,10 +139,9 @@ def is_static(path, method="GET"):
     be a file: one with a "." or ".." segment, a ";", a "\\", an encoded "/"
     or a "%xx" left after decoding is a page, so "/static/../login",
     "/static%2F..%2Flogin", "/login;x.css" and "/login%3Bx.css" all count. A
-    browser asks for its files with none of those. Nor is a path past the
-    rules' 2,000 characters: cut there, "/search/xxx...x.css/more" would end
-    in .css and flood a catch-all route past the limits. Only GET and HEAD
-    fetch a file."""
+    browser asks for its files with none of those. Nor is a path over 2,000
+    characters: no asset has a name that long. Only GET and HEAD fetch a
+    file."""
     if (method or "") not in _FETCHES:
         return False
     raw = re.split(r"[?#]", str(path or ""), 1)[0]
@@ -566,12 +565,18 @@ class Firewall:
         self.flush()
         return FLUSH_EVERY
 
-    def check(self, ip, method="", path="", user_agent="", accept="", cookie="", now=None):
+    def check(self, ip, method="", path="", user_agent="", accept="", cookie="", now=None, fetch_site=""):
         """-> {"action": "allow" | "block" | "monitor", "rule": ..., "status": ...}.
         Allowed addresses skip every other rule. accept and cookie (the
         request's headers) are for the browser check: rule "challenge",
-        status 503, answered with challenge_page()."""
+        status 503, answered with challenge_page(). fetch_site: the request's
+        Sec-Fetch-Site header."""
         allow = {"action": "allow", "rule": "", "status": 200}
+        path = origin_form(path)
+        # Another site made the browser send this (an <img> of /.env, a
+        # page asked for 120 times a minute), from the visitor's own address:
+        # refused as ever, but nobody is banned for it, here or elsewhere.
+        foreign = str(fetch_site or "").strip().lower() in ("cross-site", "same-site")
         rules = self.rules
         if not rules or not rules["enabled"]:
             return allow
@@ -583,7 +588,7 @@ class Firewall:
         if not rule and is_probe(path, rules["probes"]):
             rule = "probe"
             # Asking for a secret file: banned here in block mode; the report bans it on the other servers.
-            if action == "block":
+            if action == "block" and not foreign:
                 self._ban_here(str(ip), rules["probes"]["ban"])
         matched = None
         limits = rules.get("limits")
@@ -597,7 +602,7 @@ class Firewall:
                 # A rule the dashboard only watches passes even in block mode;
                 # one that blocks also bans the address here, as a flood does.
                 action = "block" if action == "block" and matched["action"] == "block" else "monitor"
-                if action == "block" and matched["ban"]:
+                if action == "block" and matched["ban"] and not foreign:
                     self._ban_here(str(ip), matched["ban"])
         # A static file isn't counted toward the page limits or the rate,
         # only compared with the counts so far: an address already over them
@@ -625,11 +630,16 @@ class Firewall:
                 and self._over_rate(str(ip), rules["rate"], peek=static)):
             rule = "rate"
             # Banned here straight away in block mode, as a probe is; the report bans it on the other servers.
-            if action == "block":
+            if action == "block" and not foreign:
                 self._ban_here(str(ip), rules["ban"])
         def event(name, act):
+            # Reported as only watched, so the collector bans it on no other
+            # server and counts nothing toward `shared`.
+            if foreign and act == "block" and (name in ("probe", "rate") or name.startswith("request:")):
+                act = "monitor"
             return {"ip": str(ip), "rule": name, "action": act,
-                    "method": str(method)[:10], "path": str(path).split("?")[0][:300], "user_agent": str(user_agent)[:300]}
+                    "method": str(method)[:10], "path": re.split(r"[?#]", str(path), 1)[0][:300],
+                    "user_agent": str(user_agent)[:300]}
 
         # A watched page limit is reported whatever comes after it: the rate
         # limit or the browser check deciding doesn't hide it.
@@ -694,6 +704,7 @@ class Firewall:
         beacons are never rate-limited, checked or matched against request
         rules. -> the same shape as check()."""
         allow = {"action": "allow", "rule": "", "status": 200}
+        path = origin_form(path)
         rules = self.rules
         if not rules or not rules["enabled"]:
             return allow
@@ -705,7 +716,7 @@ class Firewall:
             return allow
         action = "block" if (self.mode or rules["mode"]) == "block" else "monitor"
         self._report({"ip": str(ip), "rule": rule, "action": action, "method": str(method)[:10],
-                      "path": str(path).split("?")[0][:300], "user_agent": str(user_agent)[:300]})
+                      "path": re.split(r"[?#]", str(path), 1)[0][:300], "user_agent": str(user_agent)[:300]})
         return {"action": action, "rule": rule, "status": 403}
 
     def _needs_check(self, rules, key, ip, method, path, accept, cookie, now):

{analytics_platform-0.3.9 → analytics_platform-0.3.11}/analytics_platform/forms.py

@@ -8,7 +8,20 @@ is accepted (checked False).
 """
 import re
 
-PRIVATE_FIELD = re.compile(r"pass|card|cvc|cvv|secret|token|ssn|iban|csrf|analytics-shield", re.I)
+# Fields never sent, by name. check() is handed the whole form, so whoever
+# protects a sign-up or a checkout relies on this list: keep it wide, and the
+# same as platform-php's Forms.php, with "-" as well as "_" between words: the
+# browser's own autofill names are "cc-number", "cc-csc" and "cc-exp".
+# ("pass" also covers passwd and passport.)
+PRIVATE_FIELD = re.compile(
+    r"pass|pwd|card|cc[-_]?(num|no|number|csc|exp)|cvc|cvv|cvn|expiry|exp[-_]?(month|year|date)|secret|token"
+    r"|\bauth\b|authoriz|api[-_]?key|otp|mfa|2fa|\bpin\b|security[-_]?code|signature|private|ssn|sin\b|nino"
+    r"|tax[-_]?id|iban|bic|swift|routing|account[-_]?(no|num|number)|dob|birth|licen[cs]e|csrf|nonce"
+    r"|analytics-shield", re.I | re.A)
+# A card number in a field nobody thought to name "card" still must not be
+# sent, so values are checked too: 13-19 digits that pass Luhn are taken out
+# of the text and the rest of the message is kept, so spam is still judged.
+DIGITS = re.compile(r"\b(?:\d[ -]?){12,18}\d\b", re.A)
 MAX_FIELDS = 50
 ANSWER = "analytics-shield"
 
@@ -20,11 +33,29 @@ def plain_fields(fields):
             continue
         if isinstance(value, (list, tuple)):
             value = ", ".join(v for v in value if isinstance(v, str))
+        if isinstance(value, (int, float)) and not isinstance(value, bool):
+            value = str(value)
         if isinstance(value, str):
-            out[str(label)[:100]] = value[:2000]
+            out[str(label)[:100]] = without_card_numbers(value[:2000])
     return out
 
 
+def _luhn(digits):
+    total = 0
+    for index, char in enumerate(reversed(digits)):
+        n = int(char)
+        if index % 2:
+            n = n * 2 - 9 if n > 4 else n * 2
+        total += n
+    return total % 10 == 0
+
+
+def without_card_numbers(text):
+    """Anything shaped like a card number, and passing Luhn, as [removed].
+    Luhn is what keeps order numbers and phone numbers readable."""
+    return DIGITS.sub(lambda m: "[removed]" if _luhn(re.sub(r"\D", "", m.group(0))) else m.group(0), str(text))
+
+
 class Forms:
     def __init__(self, core, **_):
         self.core = core

{analytics_platform-0.3.9 → analytics_platform-0.3.11}/analytics_platform/proxy.py

@@ -18,6 +18,7 @@ It answers before every other module: a visitor's beacon is our own traffic,
 not a page of the site, so it shouldn't be counted, timed or rate-limited.
 """
 import hashlib
+import http.client
 import re
 import threading
 import time
@@ -56,6 +57,15 @@ FILE_PATH = re.compile(r"v(\d+(?:\.\d+\.\d+)?)/([a-z]+\.js)\Z")
 # own site. Shared cases: sdk/protocols/proxy-answer-cases.json.
 ANSWER_TYPES = frozenset(("application/json", "text/plain", "image/webp"))
 _VISIBLE = re.compile(r"[\x20-\x7e]*")
+# A query a URL can carry as it is. The visitor writes it, and http.client
+# refuses a space, a control character or a byte past 0x7f before it
+# connects: forwarded, that would count against the collector and stop
+# every visitor's beacons (the breaker).
+QUERY = re.compile(r"[\x21-\x7e]*\Z")
+# What says the collector itself failed: no connection, a timeout, a broken
+# answer. Anything else (a ValueError from text http.client won't send) is
+# this request's own fault, never the collector's.
+UNREACHABLE = (OSError, http.client.HTTPException)
 
 
 def answer_type(value):
@@ -193,10 +203,13 @@ class Proxy:
                 # The collector answered: only its own failures count against it.
                 self._broke() if answer.code >= 500 else self._worked()
                 self._remember_failure(rest)
-            except Exception as error:
+            except UNREACHABLE as error:
                 self._broke()
                 self._remember_failure(rest)
                 self.core.log("proxy", "could not fetch %s: %s" % (rest, error))
+            except Exception as error:
+                self._remember_failure(rest)
+                self.core.log("proxy", "could not fetch %s: %s" % (rest, error))
         # Upstream is unreachable: an old copy beats no tracker at all.
         if body is None and held:
             body = held[0]
@@ -220,6 +233,9 @@ class Proxy:
             return NOT_FOUND
         if body is not None and len(body) > MAX_BODY:
             return 413, [], b""
+        query = query.decode("latin-1") if isinstance(query, bytes) else str(query or "")
+        if not QUERY.match(query):
+            return 400, [("Content-Type", "text/plain; charset=utf-8")], b"Bad request"
         # The visitor's address: the collector needs it for the country, the
         # daily visitor ID and the per-visitor rate limit.
         out = {"X-Forwarded-For": client_ip or ""}
@@ -243,11 +259,14 @@ class Proxy:
         except urllib.error.HTTPError as answer:
             self._broke() if answer.code >= 500 else self._worked()
             return answer.code, self._answer_headers(answer.headers), answer.read(MAX_BODY)
-        except Exception as error:
+        except UNREACHABLE as error:
             # The tracker already treats a failed beacon as lost and carries on.
             self._broke()
             self.core.log("proxy", "could not reach the collector: %s" % error)
             return 502, [], b""
+        except Exception as error:
+            self.core.log("proxy", "could not send a beacon on: %s" % error)
+            return 400, [], b""
 
     @staticmethod
     def _answer_headers(headers):

{analytics_platform-0.3.9 → analytics_platform-0.3.11}/analytics_platform/queries.py

@@ -89,6 +89,20 @@ _NAME = frozenset("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz012345678
 _PLACEHOLDER = r"(?:\?|\$[0-9]+|:[A-Za-z0-9_]+|@[A-Za-z0-9_]+)"
 _LIST = re.compile(r"\(%s*%s(?:%s*,%s*%s)*%s*\)" % (_S, _PLACEHOLDER, _S, _S, _PLACEHOLDER, _S))
 _ROWS = re.compile(r"\(\?\)(?:%s*,%s*\(\?\))+" % (_S, _S))
+# Where "..." may hold a value rather than a name: SQLite takes a
+# double-quoted string as a literal when no column has that name (SQLCipher's
+# PRAGMA key = "x'2DD2..'"), Oracle writes a password as a quoted name, and an
+# unknown database may do either. There "..." is kept only when it reads as a
+# plain name, closed: "email", never "x'2D..'", "N3w P@ss" or an unclosed one.
+_VALUE_QUOTES = frozenset(("sqlite", "oracle", "other"))
+_QUOTED_NAME = re.compile(r'"[A-Za-z_][A-Za-z0-9_$#]{0,127}"')
+# A password or key written as a name, on any database: Oracle's
+# ALTER USER u IDENTIFIED BY pw and SQLCipher's PRAGMA key = "passphrase".
+# What follows is "?" whether it was a word or a quoted name. ASCII flags, as
+# JavaScript's \b and case-insensitive matching are.
+_SECRET_NAME = r'("(?:[^"]|"")*"?|`(?:[^`]|``)*`?|[^%s;,()]+)' % re.escape(_SPACE)
+_IDENTIFIED = re.compile(r"(\bIDENTIFIED BY )" + _SECRET_NAME, re.I | re.A)
+_PRAGMA_KEY = re.compile(r"(\bPRAGMA (?:[A-Za-z_][A-Za-z0-9_]*\.)?(?:HEX)?(?:RE)?KEY ?[=(] ?)" + _SECRET_NAME, re.I | re.A)
 _MONGO = re.compile(r"([^%s]+)(?: ([^%s]+))?(?: (.*))?\Z" % (re.escape(_SPACE), re.escape(_SPACE)), re.S)
 _INDEX_KEY = re.compile(r"(?:0|[1-9][0-9]*)\Z")
 _LONE_SURROGATE = re.compile("[\ud800-\udfff]")
@@ -178,8 +192,11 @@ def _sql_shape(sql, db, backslash_literal=None):
             last = "?"
         elif c == '"' or c == "`":
             end = _quote_end(sql, i, c, not literal)
-            out.append(sql[i:end])
-            last = sql[end - 1]
+            run = sql[i:end]
+            if c == '"' and db in _VALUE_QUOTES and not _QUOTED_NAME.fullmatch(run):
+                run = "?"
+            out.append(run)
+            last = run[-1]
             i = end
         elif c == "$" and prev not in _WORD and _DOLLAR_TAG.match(sql[i:i + 64]):
             tag = _DOLLAR_TAG.match(sql[i:i + 64]).group(0)
@@ -207,7 +224,7 @@ def _sql_shape(sql, db, backslash_literal=None):
             out.append(c)
             last = c
             i += 1
-    text = "".join(out)
+    text = _PRAGMA_KEY.sub(r"\g<1>?", _IDENTIFIED.sub(r"\g<1>?", "".join(out)))
     return _ROWS.sub("(?)", _LIST.sub("(?)", text)).strip(_SPACE)
 
 

{analytics_platform-0.3.9 → analytics_platform-0.3.11}/analytics_platform/request_rules.py

@@ -13,10 +13,15 @@ import re
 from urllib.parse import unquote_plus
 
 MAX_RULES = 200
-# How much of each part a rule may look at. Patterns come from our rules, but
-# the text comes from whoever is asking, and `re` can't be timed out: a bounded
-# slice keeps a bad pattern from being a hung request.
+# How much of a part a regex looks at at once. Patterns come from our rules,
+# but the text comes from whoever is asking, and `re` can't be timed out: a
+# bounded window keeps a bad pattern from being a hung request. The plain
+# matchers cost one pass, so they read the whole part: cut, 2,000 characters
+# of filler would carry an attack past every rule.
 MAX_TEXT = 2000
+# Each further window starts this much later, so a match up to this long is
+# found wherever it sits.
+STEP = MAX_TEXT // 2
 MAX_REGEX = 200
 SECTIONS = ("path", "query", "ua")
 # The only kinds of group a pattern may open with "(?": non-capturing and lookahead.
@@ -178,10 +183,15 @@ def _section(section, regex=True):
     pattern = section.get("regex") if regex else None
     if safe_pattern(pattern):
         try:
-            out["regex"] = re.compile(pattern, re.IGNORECASE)
+            compiled = re.compile(pattern, re.IGNORECASE)
         except re.error:
             # A pattern we can't compile is dropped; the rule's other parts stay.
-            pass
+            compiled = None
+        if compiled is not None:
+            out["regex"] = compiled
+            # "$" ends the pattern unless it is escaped: an odd run of "\" before it.
+            dollar = pattern.endswith("$") and (len(pattern) - len(pattern[:-1].rstrip("\\")) - 1) % 2 == 0
+            out["anchors"] = (pattern.startswith("^"), dollar)
     return out or None
 
 
@@ -255,9 +265,32 @@ def _router_path(path):
 
 def normalize_target(target):
     """The path of a request target (no query string) as the router reads it:
-    what the probes, the static-file check and the rules' path also match."""
+    what the probes, the static-file check and the rules' path also match.
+    Resolved from the whole path: "/./././.../login" is "/login" to a router
+    however many "./" it carries."""
     text = str(target or "")
-    return _router_path(re.split(r"[?#]", text, 1)[0][:MAX_TEXT])[:MAX_TEXT]
+    return _router_path(re.split(r"[?#]", text, 1)[0])
+
+
+def _search(regex, anchors, text):
+    """A regex over the part in windows of MAX_TEXT, each STEP after the last,
+    until one reaches the end. A pattern that begins with "^" is tried only in
+    the window that starts the part, one that ends with "$" only in the one
+    that ends it: a window's edges are not the part's."""
+    if len(text) <= MAX_TEXT:
+        return regex.search(text) is not None
+    head, tail = anchors
+    if head and tail:
+        return False
+    if head:
+        return regex.search(text[:MAX_TEXT]) is not None
+    last = max(0, -(-(len(text) - MAX_TEXT) // STEP)) * STEP
+    if tail:
+        return regex.search(text[last:]) is not None
+    for start in range(0, last + 1, STEP):
+        if regex.search(text[start:start + MAX_TEXT]):
+            return True
+    return False
 
 
 def _hits(section, raw, decoded, routed=None):
@@ -275,7 +308,7 @@ def _hits(section, raw, decoded, routed=None):
         if any(entry and entry in text for entry in section.get("contains", ())):
             return True
         regex = section.get("regex")
-        if regex is not None and regex.search(text):
+        if regex is not None and _search(regex, section["anchors"], text):
             return True
     return False
 
@@ -284,23 +317,28 @@ def prepare(method="", path="", user_agent=""):
     """The request as every rule and page limit reads it, worked out once.
     `path` is the request target as the server received it, query string and
     all. The firewall prepares a request once and hands the result to both
-    match_request and match_limits, so the path is decoded and routed once."""
+    match_request and match_limits, so the path is decoded and routed once.
+    A "#" ends the path and the query: a raw client can send one, and every
+    router drops it, so "POST /login#1" runs the login."""
     target = str(path or "")
+    end = target.find("#")
+    if end != -1:
+        target = target[:end]
     mark = target.find("?")
-    bare = (target if mark == -1 else target[:mark])[:MAX_TEXT]
-    query = ("" if mark == -1 else target[mark + 1:])[:MAX_TEXT]
+    bare = target if mark == -1 else target[:mark]
+    query = "" if mark == -1 else target[mark + 1:]
     return {
         "method": str(method or "").upper(),
         "raw_path": _normalize_path(bare),
-        "decoded_path": _normalize_path(_decode_once(bare)[:MAX_TEXT]),
+        "decoded_path": _normalize_path(_decode_once(bare)),
         # And as the router reads it: "/x;y/../admin" is "/admin" there, so a
         # rule on "/admin" catches it; a hit on any of the three counts.
-        "routed_path": _router_path(bare)[:MAX_TEXT],
+        "routed_path": _router_path(bare),
         "raw_query": query.lower(),
         # Lower-cased after decoding: %55NION decodes to "UNION", which a
         # lower-cased "union select" rule would otherwise never see.
-        "decoded_query": _decode_once(query)[:MAX_TEXT].lower(),
-        "ua": str(user_agent or "")[:MAX_TEXT].lower(),
+        "decoded_query": _decode_once(query).lower(),
+        "ua": str(user_agent or "").lower(),
     }
 
 

{analytics_platform-0.3.9 → analytics_platform-0.3.11}/analytics_platform.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: analytics-platform
-Version: 0.3.9
+Version: 0.3.11
 Summary: Firewall, error monitoring and server analytics for your Python server, run from your analytics dashboard.
 Author: steviec87
 License-Expression: MIT

{analytics_platform-0.3.9 → analytics_platform-0.3.11}/pyproject.toml

@@ -10,7 +10,7 @@ Security = "https://sitegoose.com/security"
 
 [project]
 name = "analytics-platform"
-version = "0.3.9"
+version = "0.3.11"
 description = "Firewall, error monitoring and server analytics for your Python server, run from your analytics dashboard."
 readme = "README.md"
 requires-python = ">=3.8"

{analytics_platform-0.3.9 → analytics_platform-0.3.11}/tests/test_firewall.py

@@ -42,6 +42,56 @@ class RateBan(unittest.TestCase):
         self.assertEqual(rules, ["", "", "rate", "ban"])
 
 
+class CrossSite(unittest.TestCase):
+    """A request another site made the browser send comes from the visitor's
+    own address: refused, but nobody is banned for it here or elsewhere."""
+
+    def wall(self):
+        wall = Firewall(FakeCore())
+        wall.rules = wall._compile({
+            "v": 1, "enabled": True, "mode": "block", "rate": {"limit": 2, "window": 60}, "ban": {"seconds": 600},
+            "probes": {"exact": ["/.env"], "ban": 600},
+            "requests": {"rules": [{"id": "shell", "action": "block", "ban": 600, "path": {"exact": ["/shell.php"]}}]},
+        })
+        return wall
+
+    def actions(self, wall):
+        return {row["rule"]: row["action"] for row in wall.queue.values()}
+
+    def test_a_probe_or_a_rule_is_refused_but_bans_nobody(self):
+        for value in ("cross-site", "Same-Site"):
+            wall = self.wall()
+            for path, rule in (("/.env", "probe"), ("/shell.php", "request:shell")):
+                decision = wall.check("203.0.113.9", "GET", path, fetch_site=value)
+                self.assertEqual((decision["action"], decision["rule"], decision["status"]), ("block", rule, 403))
+            self.assertEqual(wall.bans, {}, value)
+            self.assertEqual(self.actions(wall), {"probe": "monitor", "request:shell": "monitor"})
+            self.assertEqual(wall.check("203.0.113.9", "GET", "/")["rule"], "", "not banned afterwards")
+
+    def test_the_rate_is_answered_but_bans_nobody(self):
+        wall = self.wall()
+        rules = [wall.check("203.0.113.9", "GET", "/", fetch_site="cross-site")["rule"] for _ in range(4)]
+        self.assertEqual(rules, ["", "", "rate", "rate"])
+        self.assertEqual(wall.bans, {})
+        self.assertEqual(self.actions(wall), {"rate": "monitor"})
+
+    def test_same_origin_or_no_header_bans_as_before(self):
+        for value in ("same-origin", "none", ""):
+            wall = self.wall()
+            self.assertEqual(wall.check("203.0.113.9", "GET", "/.env", fetch_site=value)["rule"], "probe")
+            self.assertEqual(self.actions(wall), {"probe": "block"})
+            self.assertEqual(wall.check("203.0.113.9", "GET", "/")["rule"], "ban", value)
+
+    def test_every_adapter_hands_the_header_over(self):
+        from analytics_platform import Platform
+        platform = Platform.__new__(Platform)
+        wall = self.wall()
+        platform._modules = {"firewall": wall}
+        status, _, _ = platform.blocked("203.0.113.9", {"sec-fetch-site": "cross-site"}, "GET", "/.env")
+        self.assertEqual(status, 403)
+        self.assertEqual(wall.bans, {})
+
+
 class Cloudflare(unittest.TestCase):
     """CF-Connecting-IP is the visitor only from Cloudflare: anyone who reaches
     the origin directly can send it, and would pick their own address past

{analytics_platform-0.3.9 → analytics_platform-0.3.11}/tests/test_hardening.py

@@ -546,5 +546,28 @@ class SharedKeysAndHeaders(unittest.TestCase):
         for name in ("Content-Length", "transfer-encoding", "Connection", "Set-Cookie", "Content-Type"):
             self.assertFalse(safe_header([name, "1"]), name)
 
+
+class PrivateFields(unittest.TestCase):
+    """Form protection is handed the whole form, sign-ups and checkouts
+    included: the same private names as platform-php, and card numbers
+    taken out of any value."""
+
+    def test_private_names_are_never_sent(self):
+        from analytics_platform.forms import plain_fields
+        private = ("password", "pwd", "pin", "otp", "api_key", "apiKey", "cc-number", "cc_num", "cc-csc", "cc-exp",
+                   "exp_month", "security_code", "iban", "routing", "account_number", "dob", "nonce", "auth",
+                   "two_factor_mfa", "csrfmiddlewaretoken", "analytics-shield")
+        kept = plain_fields({name: "x" for name in private + ("email", "message", "spinach")})
+        self.assertEqual(sorted(kept), ["email", "message", "spinach"])
+
+    def test_a_card_number_in_any_value_is_removed(self):
+        from analytics_platform.forms import plain_fields, without_card_numbers
+        self.assertEqual(without_card_numbers("pay with 4111 1111 1111 1111 please"), "pay with [removed] please")
+        self.assertEqual(without_card_numbers("4111-1111-1111-1111"), "[removed]")
+        # Luhn keeps order and phone numbers readable.
+        self.assertEqual(without_card_numbers("order 1234567890123"), "order 1234567890123")
+        self.assertEqual(plain_fields({"notes": ["5555555555554444", 7]}), {"notes": "[removed]"})
+        self.assertEqual(plain_fields({"qty": 3}), {"qty": "3"})
+
 if __name__ == "__main__":
     unittest.main()

{analytics_platform-0.3.9 → analytics_platform-0.3.11}/tests/test_proxy.py

@@ -137,6 +137,19 @@ class ProxyTest(unittest.TestCase):
         self.assertEqual(status, 404)
         self.assertEqual(Collector.seen, [])
 
+    def test_a_query_no_url_can_carry_is_refused_and_never_trips_the_breaker(self):
+        # http.client would raise on these before connecting; counted as the
+        # collector failing, a few a second would stop every visitor's beacons.
+        kind, rest = self.proxy.match(self.proxy.collect_url())
+        for query in ("x=\xff", "x=\x00", "x=a b", "x=\u00e9"):
+            status, _, _ = self.proxy.handle(kind, rest, method="GET", query=query)
+            self.assertEqual(status, 400, repr(query))
+        self.assertEqual(Collector.seen, [])
+        self.assertEqual(self.proxy._failures, 0)
+        self.assertFalse(self.proxy.resting())
+        status, _, _ = self.proxy.handle(kind, rest, method="GET", query="v=1&u=%2Fa")
+        self.assertEqual(status, 202)
+
     def test_switched_off_it_answers_nothing(self):
         off = Proxy(FakeCore(self.endpoint), enabled=False)
         self.assertIsNone(off.match(off.script_url()))

{analytics_platform-0.3.9 → analytics_platform-0.3.11}/tests/test_request_rules.py

@@ -51,6 +51,55 @@ class SharedCases(unittest.TestCase):
         self.assertEqual(len(compile_rules({"rules": many})), 200)
 
 
+class OriginForm(unittest.TestCase):
+    """The request line's target as the router reads it: gunicorn routes
+    "x:/admin" as "/admin", so the firewall must match it as "/admin" too."""
+
+    def test_every_shared_target(self):
+        from analytics_platform.core import origin_form
+        with open(CASES) as handle:
+            targets = json.load(handle)["targets"]
+        self.assertTrue(targets)
+        for case in targets:
+            self.assertEqual(origin_form(case["target"]), case["origin"], case["target"])
+
+    def test_the_firewall_matches_the_routed_target(self):
+        from analytics_platform.core import origin_form
+        rules = compile_rules({"rules": [{"id": "admin", "action": "block", "path": {"prefix": ["/admin"]}}]})
+        self.assertEqual(match_request(rules, "POST", origin_form("x:/admin/login/"), "")["id"], "admin")
+
+
+class RegexWindows(unittest.TestCase):
+    """Only a regex is held to 2,000 characters at a time; a window's edges
+    are not the part's, so anchors hold only where the part really starts
+    or ends."""
+
+    def rules(self, pattern):
+        return compile_rules({"rules": [{"id": "r", "action": "block", "query": {"regex": pattern}}]})
+
+    def test_found_past_the_first_window(self):
+        target = "/x?" + "a" * 5000 + "union select"
+        self.assertEqual(match_request(self.rules("union\\s+select"), "GET", target, "")["id"], "r")
+
+    def test_a_caret_only_at_the_start(self):
+        rules = self.rules("^pad")
+        self.assertEqual(match_request(rules, "GET", "/x?pad" + "a" * 5000, "")["id"], "r")
+        # 1,000 characters in is a window's start, never the query's.
+        self.assertIsNone(match_request(rules, "GET", "/x?" + "a" * 1000 + "pad" + "a" * 5000, ""))
+
+    def test_a_dollar_only_at_the_end(self):
+        rules = self.rules("end$")
+        self.assertEqual(match_request(rules, "GET", "/x?" + "a" * 5000 + "end", "")["id"], "r")
+        self.assertIsNone(match_request(rules, "GET", "/x?" + "a" * 1997 + "end" + "a" * 3000, ""))
+        # An escaped "$" is a character, not an anchor.
+        self.assertEqual(match_request(self.rules("end\\$"), "GET", "/x?" + "a" * 1997 + "end$" + "a" * 3000, "")["id"], "r")
+
+    def test_both_anchors_only_on_a_short_part(self):
+        rules = self.rules("^a+$")
+        self.assertEqual(match_request(rules, "GET", "/x?" + "a" * 2000, "")["id"], "r")
+        self.assertIsNone(match_request(rules, "GET", "/x?" + "a" * 2001, ""))
+
+
 class PatternSafety(unittest.TestCase):
     """(a+)+ takes exponential time on input that nearly matches, and `re`
     can't be timed out: the client must refuse the pattern, not match it."""