proxy2model 2.0.0__tar.gz → 2.0.1__tar.gz

{proxy2model-2.0.0/src/proxy2model.egg-info → proxy2model-2.0.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxy2model
-Version: 2.0.0
+Version: 2.0.1
 Summary: Turn captured proxy traffic into an evidence-backed model of a web app.
 Author: falc0n
 License-Expression: Apache-2.0

{proxy2model-2.0.0 → proxy2model-2.0.1}/src/proxy2model/__init__.py

@@ -1,5 +1,5 @@
 """proxy2model — turn captured proxy traffic into an evidence-backed web-app model."""
-__version__ = "2.0.0"
+__version__ = "2.0.1"
 
 import os as _os
 

{proxy2model-2.0.0 → proxy2model-2.0.1}/src/proxy2model/cli/browse.py

@@ -226,6 +226,7 @@ def cmd_browse(args) -> int:
     if recorder.error:
         print(ui.warn(f"recording stopped early: {recorder.error}"), file=sys.stderr)
     live.flush()
+    live.finish()
     if not live.saved:
         print(f"\n  nothing in scope was recorded ({', '.join(scope)})")
         return 1

{proxy2model-2.0.0 → proxy2model-2.0.1}/src/proxy2model/ingest/live.py

@@ -2,7 +2,9 @@
 Live capture: exchanges recorded while you browse are masked as they arrive, appended to
 `inputs/browse.jsonl`, and the model is rebuilt every few seconds while anything new comes in.
 
-Only the masked record is ever written; a restart continues where the last session stopped.
+Only the masked record is ever written; a restart continues where the last session stopped. The cookie
+and credential values seen during the session are held in memory only, so a value a page script sent
+before its cookie was set is masked when the batch is saved, or at the latest when the session ends.
 """
 
 from __future__ import annotations
@@ -13,7 +15,7 @@ import threading
 import time
 from xml.etree.ElementTree import Element
 
-from .parse import Exchange, parse_elements
+from .parse import Exchange, _scrubber, parse_elements
 
 
 class Live:
@@ -27,6 +29,7 @@ class Live:
         self.stats = {"items": 0, "parsed": 0, "skipped": 0}
         self.received = 0
         self.saved = 0
+        self.known: set[str] = set()                 # raw values, never written
         if os.path.exists(path):
             with open(path, encoding="utf-8") as f:
                 self.saved = sum(1 for line in f if line.strip() and '"_stats"' not in line[:12])
@@ -34,7 +37,7 @@ class Live:
     def ingest(self, items: list[Element]) -> tuple[int, int]:
         """Parse and mask each recorded <item>; returns (accepted, skipped)."""
         stats: dict = {}
-        parsed = list(parse_elements(items, source="live", role=self.role, stats=stats))
+        parsed = list(parse_elements(items, source="live", role=self.role, stats=stats, known=self.known))
         skipped = len(items) - len(parsed)
         with self.lock:
             for ex in parsed:
@@ -53,7 +56,8 @@ class Live:
             self.pending, self.stats = [], {"items": 0, "parsed": 0, "skipped": 0}
         if not batch and not stats["items"]:
             return 0
-        lines = [json.dumps(ex.to_record()) for ex in batch]
+        mask = self._mask()
+        lines = [mask(json.dumps(ex.to_record())) for ex in batch]
         lines.append(json.dumps({"_stats": stats, "_version": 1}))
         os.makedirs(os.path.dirname(self.path), exist_ok=True)
         with open(self.path, "a", encoding="utf-8") as f:
@@ -63,6 +67,28 @@ class Live:
         self.saved += len(batch)
         return len(batch)
 
+    def _mask(self):
+        """Mask every value learned so far, as it appears inside a JSON-encoded record."""
+        with self.lock:
+            values = list(self.known)
+        return _scrubber({json.dumps(v)[1:-1] for v in values} | set(values))
+
+    def finish(self) -> None:
+        """Mask, in what was saved earlier, the values learned later in the session."""
+        if not self.known or not os.path.exists(self.path):
+            return
+        mask = self._mask()
+        with open(self.path, encoding="utf-8") as f:
+            text = f.read()
+        masked = mask(text)
+        if masked != text:
+            tmp = self.path + ".tmp"
+            with open(tmp, "w", encoding="utf-8") as f:
+                f.write(masked)
+                f.flush()
+                os.fsync(f.fileno())
+            os.replace(tmp, self.path)
+
 
 class Rebuilder(threading.Thread):
     """Every `interval` seconds, saves what arrived and, if anything did, rebuilds.

{proxy2model-2.0.0 → proxy2model-2.0.1}/src/proxy2model/ingest/parse.py

@@ -22,6 +22,7 @@ from __future__ import annotations
 
 import base64
 import codecs
+import functools
 import json
 import os
 import re
@@ -455,16 +456,18 @@ def _exchange_secrets(req_headers, resp_headers) -> list[str]:
     return sorted((x for x in out if len(x) >= 8), key=len, reverse=True)
 
 
-def _scrubber(values: list[str]):
+def _scrubber(values) -> "callable":
+    """Mask every one of `values` wherever it appears in a text: one regex pass, longest first."""
+    values = tuple(sorted(set(values), key=len, reverse=True))
     if not values:
         return lambda text: text
+    return _compiled_scrubber(values)
 
-    def scrub(text: str) -> str:
-        for v in values:
-            if v in text:
-                text = text.replace(v, REDACTED)
-        return text
-    return scrub
+
+@functools.lru_cache(maxsize=8)
+def _compiled_scrubber(values: tuple):
+    rx = re.compile("|".join(map(re.escape, values)))
+    return lambda text: rx.sub(REDACTED, text) if text else text
 
 
 def _set_cookies(resp_headers) -> list[dict]:
@@ -580,18 +583,28 @@ def parse_items(path: str, role: str | None = None, stats: dict | None = None,
         stats = {}
     for k in ("items", "parsed", "skipped"):
         stats.setdefault(k, 0)
+    # A page script can send a value to a vendor (`fbp=…` in a URL, `visUuid="…"` in a body) before
+    # the cookie that carries it is ever set. So the file is read twice: first every cookie and
+    # credential value is collected from the headers, then each is masked wherever it appears.
+    known: set[str] = set()
+    for _ in _parse_path(path, role, {"items": 0, "parsed": 0, "skipped": 0}, (), (), known, True):
+        pass
+    yield from _parse_path(path, role, stats, skip_tools, only_tools, known, False)
+
+
+def _parse_path(path, role, stats, skip_tools, only_tools, known, collect):
     if path.lower().endswith(".csv"):
         caido = _caido_csv_header(path)
         if caido is not None:
-            yield from _parse_caido(path, role, stats, skip_tools, only_tools, caido)
+            yield from _parse_caido(path, role, stats, skip_tools, only_tools, caido, known, collect)
             return
-        yield from _parse_logger_csv(path, role, stats, skip_tools, only_tools)
+        yield from _parse_logger_csv(path, role, stats, skip_tools, only_tools, known, collect)
         return
     if _looks_like_caido_json(path):
-        yield from _parse_caido(path, role, stats, skip_tools, only_tools)
+        yield from _parse_caido(path, role, stats, skip_tools, only_tools, None, known, collect)
         return
     if path.lower().endswith(".har") or _looks_like_har(path):
-        yield from _parse_har(path, role, stats)
+        yield from _parse_har(path, role, stats, known, collect)
         return
     _check_no_entities(path)
     source = os.path.basename(path)
@@ -607,7 +620,7 @@ def parse_items(path: str, role: str | None = None, stats: dict | None = None,
             continue
         stats["items"] += 1
         try:
-            ex = _parse_item(elem, idx, role)
+            ex = _parse_item(elem, idx, role, known, collect)
             if ex is None:
                 stats["skipped"] += 1
             else:
@@ -687,7 +700,7 @@ def _b64_text(value: str) -> tuple[str, bool]:
     return "true", re.sub(r"\s+", "", v)
 
 
-def _parse_logger_csv(path, role, stats, skip_tools, only_tools):
+def _parse_logger_csv(path, role, stats, skip_tools, only_tools, known=None, collect=False):
     source = os.path.basename(path)
     skip = {t.strip().lower() for t in skip_tools if t.strip()}
     only = {t.strip().lower() for t in only_tools if t.strip()}
@@ -720,7 +733,7 @@ def _parse_logger_csv(path, role, stats, skip_tools, only_tools):
             # a response that stops at the spreadsheet cell limit and never recovered is cut short
             if not row.get("_spilled") and _CELL_LIMIT <= len(row.get("Response") or "") < 32768:
                 SubElement(item, "truncated").text = "1"
-            ex = _parse_item(item, idx, role)
+            ex = _parse_item(item, idx, role, known, collect)
         except Exception:
             ex = None
         if ex is None:
@@ -924,17 +937,17 @@ def _har_entries(f, chunk: int = 1 << 20):
         raise ValueError("not a HAR file (no log.entries)")
 
 
-def _parse_har(path, role, stats):
+def _parse_har(path, role, stats, known=None, collect=False):
     """A HAR file (browser DevTools, Hetty, mitmproxy, ZAP, Charles ...), read as if it were a Burp export.
 
     Streamed: entries are read and turned into exchanges one at a time, never the whole file at once.
     """
     source = os.path.basename(path)
     with open(path, encoding="utf-8-sig", errors="replace") as f:
-        yield from _har_exchanges(_har_entries(f), source, role, stats)
+        yield from _har_exchanges(_har_entries(f), source, role, stats, known, collect)
 
 
-def _har_exchanges(entries, source, role, stats):
+def _har_exchanges(entries, source, role, stats, known=None, collect=False):
     idx = 1
     item_no = -1
     while True:
@@ -948,7 +961,7 @@ def _har_exchanges(entries, source, role, stats):
         stats["items"] += 1
         try:
             item = _har_item(entry) if isinstance(entry, dict) else None
-            ex = _parse_item(item, idx, role) if item is not None else None
+            ex = _parse_item(item, idx, role, known, collect) if item is not None else None
         except Exception:
             ex = None
         if ex is None:
@@ -1039,7 +1052,7 @@ def _caido_records(path: str, csv_header: list[str] | None):
             raise ValueError(f"not a Caido export (not a valid JSON array: {e})")
 
 
-def _parse_caido(path, role, stats, skip_tools, only_tools, csv_header=None):
+def _parse_caido(path, role, stats, skip_tools, only_tools, csv_header=None, known=None, collect=False):
     """A Caido HTTP history export (JSON or CSV), read as if it were a Burp export. Streamed.
 
     `source` (intercept, replay, automate ...) plays the part of Logger++'s Tool column for
@@ -1064,7 +1077,7 @@ def _parse_caido(path, role, stats, skip_tools, only_tools, csv_header=None):
         stats["items"] += 1
         try:
             item = _caido_item(rec)
-            ex = _parse_item(item, idx, role) if item is not None else None
+            ex = _parse_item(item, idx, role, known, collect) if item is not None else None
         except Exception:
             ex = None
         if ex is None:
@@ -1076,13 +1089,23 @@ def _parse_caido(path, role, stats, skip_tools, only_tools, csv_header=None):
         yield ex
 
 
-def parse_elements(items, source: str, role: str | None = None, stats: dict | None = None):
+def parse_elements(items, source: str, role: str | None = None, stats: dict | None = None,
+                   known: set | None = None):
     """Like parse_items, for <item> elements already in memory (the crawler's output).
 
     The crawler hands over what it fetched without ever writing a raw
     request/response to disk; redaction still happens here, before an Exchange
-    leaves this module.
+    leaves this module. `known` keeps the cookie and credential values seen so far (pass the same
+    set for every batch of one session); the batch is read for them before any of it is masked.
     """
+    if known is None:
+        known = set()
+    items = list(items)
+    for elem in items:
+        try:
+            _parse_item(elem, 0, role, known, True)
+        except Exception:
+            pass
     if stats is None:
         stats = {}
     for k in ("items", "parsed", "skipped"):
@@ -1091,7 +1114,7 @@ def parse_elements(items, source: str, role: str | None = None, stats: dict | No
     for item_no, elem in enumerate(items):
         stats["items"] += 1
         try:
-            ex = _parse_item(elem, idx, role)
+            ex = _parse_item(elem, idx, role, known)
         except Exception:
             ex = None
         if ex is None:
@@ -1108,7 +1131,10 @@ def _int(text: str | None, default=None):
     return int(t) if t.isdigit() else default
 
 
-def _parse_item(elem, idx: int, role: str | None) -> Exchange | None:
+def _parse_item(elem, idx: int, role: str | None, known: set | None = None, collect: bool = False) -> Exchange | None:
+    """One <item> as a masked Exchange. `known` holds cookie and credential values seen anywhere in the
+    capture; each is masked wherever it appears, not only beside its own name. With `collect`, the item
+    only adds its values to `known` and nothing is built."""
     method = (_text(elem.find("method")) or "GET").strip().upper()
     host = (_text(elem.find("host")) or "").strip().lower()
     port = _int(_text(elem.find("port")), 0)
@@ -1134,6 +1160,11 @@ def _parse_item(elem, idx: int, role: str | None) -> Exchange | None:
     resp_head, resp_body_b = _split_http(resp_raw) if resp_raw else ("", b"")
     req_start, req_headers = _parse_headers(req_head)
     resp_start, resp_headers = _parse_headers(resp_head)
+    own_secrets = _exchange_secrets(req_headers, resp_headers)
+    if known is not None:
+        known.update(own_secrets)
+        if collect:
+            return None
     if status is None and resp_start.startswith("HTTP/"):
         status = _int(resp_start.split(" ")[1] if " " in resp_start else None)
 
@@ -1156,7 +1187,7 @@ def _parse_item(elem, idx: int, role: str | None) -> Exchange | None:
     preflight = method == "OPTIONS" and bool(_header(req_headers, "access-control-request-method"))
 
     secrets: list[SecretFingerprint] = []
-    scrub = _scrubber(_exchange_secrets(req_headers, resp_headers))
+    scrub = _scrubber(known if known else own_secrets)
     red_path, s0 = redact_path(path)
     red_path = scrub(red_path)
     ptemplate = route_template(red_path, template_path(red_path))

{proxy2model-2.0.0 → proxy2model-2.0.1}/src/proxy2model/ingest/redact.py

@@ -55,7 +55,10 @@ SENSITIVE_PARAM_HINTS = (
     # WordPress/Django/.NET nonces, __VIEWSTATE and friends
     "sessid", "nonce", "xsrf", "viewstate", "eventvalidation", "verification",
     # a value that is itself a credential, whatever the field is called around it
-    "cookie", "credential", "bearer",
+    "cookie", "credential", "bearer", "hmac",
+    # personal data with no shape to catch it by: a phone number is just ten digits
+    "phone", "mobile", "msisdn", "tel", "whatsapp", "email", "dob", "birth", "aadhaar", "aadhar",
+    "passport", "pan", "gstin", "ssn",
 )
 
 # Names that are innocuous alone (`new=true`) but are the password fields of a
@@ -250,6 +253,8 @@ def redact_body(body: str) -> tuple[str, list[SecretFingerprint]]:
 # `accessToken` and `user[password]` are caught, not just the exact hints.
 _KV_ANY = re.compile(r"(^|[?&;,#\s])([A-Za-z0-9_.\-\[\]]{1,64})(\s*=\s*)([^&;#\s\"'<>]+)")
 _JSON_ANY = re.compile(r'"((?:[^"\\]|\\.){1,64})"(\s*:\s*)("(?:[^"\\]|\\.)*"|-?\d[\d.eE+-]*|true|false|null)')
+# JSON kept inside a script string, its quotes escaped: var cfg = "{\"token\":\"…\"}"
+_JSON_ESC = re.compile(r'\\"([A-Za-z0-9_.$@\-\[\]]{1,64})\\"(\s*:\s*)(\\"(?:(?!\\").){0,500}\\"|-?\d[\d.eE+-]*|true|false|null)')
 
 
 _INPUT_TAG = re.compile(r"<input\b[^>]*>", re.I)
@@ -281,7 +286,8 @@ def redact_named(text: str) -> str:
         return text
 
     names = [m.group(2) for m in _KV_ANY.finditer(text)] + \
-            [m.group(1) for m in _JSON_ANY.finditer(text)]
+            [m.group(1) for m in _JSON_ANY.finditer(text)] + \
+            ([m.group(1) for m in _JSON_ESC.finditer(text)] if '\\"' in text else [])
     group = _is_credential_group(names)
     sens = lambda n: _is_sensitive_param(n) or (group and _leaf(n) in _CREDENTIAL_GROUP)
 
@@ -330,6 +336,15 @@ def redact_named(text: str) -> str:
     text = _KV_ANY.sub(_kv, text)
     text = _JSON_ANY.sub(_json, text)
 
+    def _json_esc(m: re.Match) -> str:
+        val = m.group(3)[2:-2] if m.group(3).startswith('\\"') else m.group(3)
+        if sens(m.group(1)) and REDACTED not in val and not _is_error_constant(val):
+            return '\\"' + m.group(1) + '\\"' + m.group(2) + '\\"' + REDACTED + '\\"'
+        return m.group(0)
+
+    if '\\"' in text:
+        text = _JSON_ESC.sub(_json_esc, text)
+
     def _live(v: str) -> bool:
         return bool(v) and REDACTED not in v and not _is_error_constant(v)
 

{proxy2model-2.0.0 → proxy2model-2.0.1/src/proxy2model.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: proxy2model
-Version: 2.0.0
+Version: 2.0.1
 Summary: Turn captured proxy traffic into an evidence-backed model of a web app.
 Author: falc0n
 License-Expression: Apache-2.0

{proxy2model-2.0.0 → proxy2model-2.0.1}/src/proxy2model.egg-info/SOURCES.txt

@@ -85,6 +85,7 @@ tests/test_bql_parity.py
 tests/test_browse.py
 tests/test_browser_crawl.py
 tests/test_caido.py
+tests/test_capture_wide_masking.py
 tests/test_cli.py
 tests/test_crawl.py
 tests/test_diff.py

proxy2model-2.0.1/tests/test_capture_wide_masking.py

@@ -0,0 +1,93 @@
+"""A cookie or credential value is masked wherever it appears in the capture, not only beside its name.
+
+Found on a real Caido capture: page scripts send tracking ids to vendors (`fbp=…` in a URL,
+`visUuid="…"` in a page) before the cookie that carries them is set, and an OTP request carried a
+phone number inside escaped JSON.
+"""
+
+import json
+import os
+from xml.etree.ElementTree import Element, SubElement
+
+from proxy2model import parse_items
+from proxy2model.cli import main
+from proxy2model.ingest.live import Live
+from proxy2model.ingest.redact import redact_named
+
+from test_caido import b64, caido_json, rec
+
+FBP = "fb.1.1793913083850.123456789012345"
+VWO = "D2DFAA0B1C2D3E4F5A6B7C8D9E0F1A2B3"
+
+
+def outputs(out):
+    text = ""
+    for root, _, files in os.walk(out):
+        for f in files:
+            text += open(os.path.join(root, f), encoding="utf-8", errors="replace").read()
+    return text
+
+
+CAPTURE = [
+    # the vendor gets the id first, in its query string and in the page's inline script
+    rec(1, "GET", "/tr", 200, query=f"id=1&fbp={FBP}&ev=PageView"),
+    rec(2, "GET", "/", 200, rs_head="Content-Type: text/html\r\n",
+        rs_body=f'<html><script>window.VWO.visUuid="{VWO}";</script></html>'),
+    # only later does a request carry them as cookies
+    rec(3, "GET", "/api/items", 200, rq_head=f"Cookie: _fbp={FBP}; _vwo_uuid_v2={VWO}\r\n"),
+    rec(4, "POST", "/v4-sec/notification/send", 200, rq_head="Content-Type: application/json\r\n",
+        rq_body=json.dumps({"payload": json.dumps({"recipient": {"callingCode": "91", "mobile": "9990001112"},
+                                                   "hmac": "8922d8cd5d40ab410ee25008ec1aa47d"})})),
+]
+
+
+def test_a_value_sent_before_its_cookie_is_masked_everywhere(tmp_path, monkeypatch):
+    monkeypatch.setenv("PROXY2MODEL_FP_KEY", "ci")
+    monkeypatch.setenv("PROXY2MODEL_OFFLINE", "1")
+    out = tmp_path / "o"
+    assert main([caido_json(tmp_path, CAPTURE), "-w", "shop", "--out", str(out)]) == 0
+    text = outputs(out)
+    for secret in (FBP, VWO, "9990001112", "8922d8cd5d40ab410ee25008ec1aa47d"):
+        assert secret not in text, secret
+
+
+def test_the_model_is_the_same_apart_from_the_masking(tmp_path):
+    got = list(parse_items(caido_json(tmp_path, CAPTURE)))
+    assert [e.method for e in got] == ["GET", "GET", "GET", "POST"]
+    assert any(k == "fbp" for k, *_ in got[0].query_params)                 # the name stays, the value goes
+
+
+def test_escaped_json_in_a_script_is_masked_by_name():
+    text = r'var c="{\"token\":\"s3cr3tvalue123456\",\"password\":\"hunter2hunter2\",\"name\":\"bob\"}";'
+    out = redact_named(text)
+    assert "s3cr3tvalue123456" not in out and "hunter2hunter2" not in out
+    assert r'\"name\":\"bob\"' in out                                          # an ordinary field is left alone
+
+
+def test_personal_fields_are_masked_by_name():
+    out = redact_named('{"mobile":"9990001112","phone_no":"9990001113","dob":"1990-01-01","panel":"left"}')
+    for v in ("9990001112", "9990001113", "1990-01-01"):
+        assert v not in out
+    assert '"panel":"left"' in out
+
+
+def item(path, cookie=""):
+    el = Element("item")
+    for tag, val in (("method", "GET"), ("host", "shop.test"), ("port", "443"), ("protocol", "https"),
+                     ("path", path), ("status", "200"), ("mimetype", "")):
+        SubElement(el, tag).text = val
+    rq = f"GET {path} HTTP/1.1\r\nHost: shop.test\r\n" + (f"Cookie: {cookie}\r\n" if cookie else "") + "\r\n"
+    SubElement(el, "request", {"base64": "true"}).text = b64(rq)
+    SubElement(el, "response", {"base64": "true"}).text = b64("HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\r\n{}")
+    return el
+
+
+def test_browse_masks_what_it_saved_before_the_cookie_appeared(tmp_path):
+    live = Live(str(tmp_path / "inputs" / "browse.jsonl"))
+    live.ingest([item(f"/tr?fbp={FBP}")])
+    live.flush()                                                              # saved while the value was unknown
+    live.ingest([item("/api/items", cookie=f"_fbp={FBP}")])
+    live.flush()
+    live.finish()
+    text = open(tmp_path / "inputs" / "browse.jsonl").read()
+    assert FBP not in text and "fbp" in text