proxy2model 2.0.0__py3-none-any.whl

proxy2model/ingest/extract.py

@@ -0,0 +1,208 @@
+"""
+Static endpoint extraction from JavaScript and HTML bodies.
+
+A reference found in code is a *claim* that the app may call an endpoint, not
+proof that it does, so everything extracted here becomes an INFERRED edge in
+the model. Each reference is a (method, host, path, kind) tuple:
+
+  method  "GET", "POST", ... when the call site names it, else None
+  host    the host of an absolute URL, else None (relative to the page origin)
+  path    templated the same way runtime paths are (`/users/{id}`), so a
+          reference and a request for the same endpoint reconcile
+  kind    "call" (fetch/axios/XHR/verb call), "string" (bare API-looking
+          string literal) or "form" (an HTML <form action>)
+"""
+
+from __future__ import annotations
+
+import re
+import urllib.parse
+
+from .redact import template_path
+
+# Bodies larger than this are scanned only up to the cap and flagged.
+MAX_SCAN_BYTES = 10 * 1024 * 1024
+
+# Host of a reference whose base URL the code keeps in a variable (`${apiHost}/v1/accounts`):
+# the path is known, the host is not, so it is never pinned to the page's own host.
+VAR_BASE = "{base}"
+
+API_HINT = re.compile(r"/(api|v\d+|graphql|rest|gql)(/|$)", re.I)
+
+# `${expr}` inside template literals becomes a {param} placeholder.
+_TEMPLATE_EXPR_RE = re.compile(r"\$\{[^{}`]{0,120}\}")
+
+# A URL-ish literal: optional scheme+host, then a path. `{param}` may appear
+# (from template literals) anywhere, including as a leading base-URL prefix.
+_URL_BODY = r"(?:\{param\})?(?:https?:)?/[^\s\"'`<>\\]{0,300}?"
+
+_VERB_CALL_RE = re.compile(
+    r"\.(get|post|put|patch|delete|head)\s*\(\s*([\"'`])(" + _URL_BODY + r")\2", re.I)
+_FETCH_RE = re.compile(
+    r"(?:\b(?:fetch|axios|ajax|request)|\$http)\s*\(\s*([\"'`])(" + _URL_BODY + r")\1")
+_XHR_OPEN_RE = re.compile(
+    r"\.open\s*\(\s*[\"'](GET|POST|PUT|PATCH|DELETE|HEAD|OPTIONS)[\"']\s*,\s*"
+    r"([\"'`])(" + _URL_BODY + r")\2", re.I)
+_METHOD_NEAR_RE = re.compile(r"""method\s*:\s*["'`](GET|POST|PUT|PATCH|DELETE|HEAD)["'`]""", re.I)
+_STRING_RE = re.compile(r"([\"'`])(" + _URL_BODY + r")\1")
+
+_FORM_RE = re.compile(r"<form\b([^>]*)>", re.I)
+_ATTR_RE = re.compile(r"""\b(action|method)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))""", re.I)
+
+_STATIC_EXT = re.compile(
+    r"\.(?:png|jpe?g|gif|svg|ico|webp|avif|css|woff2?|ttf|otf|eot|map|mp4|webm|mp3|"
+    r"html?|js|mjs|json|txt|md)$", re.I)
+
+
+def _normalize(raw: str) -> tuple[str | None, str] | None:
+    """Split a raw literal into (host, templated path); None if not a path."""
+    raw = raw.strip()
+    var_base = raw.startswith("{param}")
+    if var_base:
+        raw = raw[len("{param}"):]          # the base URL is a variable: the host is unknown
+    if raw.startswith("//"):
+        raw = "https:" + raw
+    host = None
+    if raw.lower().startswith(("http://", "https://")):
+        try:
+            parts = urllib.parse.urlsplit(raw)
+        except ValueError:                   # `https://[{x}]/…` in minified code is not a URL
+            return None
+        host = (parts.hostname or "").lower() or None
+        path = parts.path or "/"
+    else:
+        if not raw.startswith("/"):
+            return None
+        path = raw.split("?", 1)[0].split("#", 1)[0]
+    if "//" in path or len(path) > 200:
+        return None
+    # `"/api/users/" + id` — a trailing slash is a concatenation point
+    if len(path) > 1 and path.endswith("/"):
+        path = path + "{param}"
+    return host or (VAR_BASE if var_base else None), template_path(path)
+
+
+def _ref(method, raw, kind, out: set) -> None:
+    norm = _normalize(raw)
+    if norm is None:
+        return
+    host, path = norm
+    if path == "/" and host in (None, VAR_BASE):
+        return
+    # `${this.host}/${id}/reviews` — the base is a service prefix the bundle
+    # keeps in a variable, so all that is left is `/{param}/reviews`. That is
+    # not a path anything can be matched to; drop it rather than invent a
+    # root-level endpoint (an API-looking segment after it keeps it: `/{t}/api/x`).
+    if host in (None, VAR_BASE) and path.startswith("/{param}") and not API_HINT.search(path):
+        return
+    out.add((method.upper() if method else None, host, path, kind))
+
+
+def extract_js_refs(body: str) -> set[tuple[str | None, str | None, str, str]]:
+    """Endpoint references inside JavaScript (or inline <script>) text."""
+    out: set = set()
+    if not body:
+        return out
+    body = _TEMPLATE_EXPR_RE.sub("{param}", body)
+    call_spans: list[tuple[int, int]] = []
+
+    for m in _XHR_OPEN_RE.finditer(body):
+        _ref(m.group(1), m.group(3), "call", out)
+        call_spans.append(m.span(3))
+    for m in _VERB_CALL_RE.finditer(body):
+        _ref(m.group(1), m.group(3), "call", out)
+        call_spans.append(m.span(3))
+    for m in _FETCH_RE.finditer(body):
+        # fetch(url, {method: "POST"}) — read the method only from THIS call's
+        # options object, i.e. before the statement ends or the next call begins,
+        # so a later call's method can't bleed onto this one.
+        window = body[m.end(): m.end() + 200]
+        stop = len(window)
+        for tok in (";", "fetch(", "axios", ".open(", ".then(", "\n\n"):
+            i = window.find(tok)
+            if i != -1:
+                stop = min(stop, i)
+        near = _METHOD_NEAR_RE.search(window[:stop])
+        _ref(near.group(1) if near else None, m.group(2), "call", out)
+        call_spans.append(m.span(2))
+
+    called = {s for s in call_spans}
+    for m in _STRING_RE.finditer(body):
+        if m.span(2) in called:
+            continue
+        raw = m.group(2)
+        norm = _normalize(raw)
+        if norm is None:
+            continue
+        _, path = norm
+        # a bare string is only interesting if it looks like an API route
+        if API_HINT.search(path) and not _STATIC_EXT.search(path):
+            _ref(None, raw, "string", out)
+    return out
+
+
+def extract_html_refs(body: str) -> set[tuple[str | None, str | None, str, str]]:
+    """<form action> targets plus references inside inline scripts."""
+    out: set = set()
+    if not body:
+        return out
+    for fm in _FORM_RE.finditer(body):
+        attrs = {}
+        for am in _ATTR_RE.finditer(fm.group(1)):
+            attrs[am.group(1).lower()] = am.group(2) or am.group(3) or am.group(4) or ""
+        action = attrs.get("action", "").strip()
+        if not action or action.startswith(("#", "javascript:", "mailto:")):
+            continue
+        method = (attrs.get("method") or "GET").upper()
+        _ref(method if method in ("GET", "POST") else "GET", action, "form", out)
+    out |= extract_js_refs(body)
+    return out
+
+
+# ---- source maps and workers -------------------------------------------------
+
+_SOURCEMAP_RE = re.compile(r"//[#@]\s*sourceMappingURL=([^\s'\"]+)")
+_WORKER_RES = (
+    ("worker", re.compile(r"new\s+(?:Shared)?Worker\s*\(\s*([\"'`])([^\"'`\s]{1,200})\1")),
+    ("service_worker", re.compile(r"serviceWorker\s*\.\s*register\s*\(\s*([\"'`])([^\"'`\s]{1,200})\1")),
+    ("import_scripts", re.compile(r"importScripts\s*\(\s*([\"'`])([^\"'`\s]{1,200})\1")),
+)
+
+
+def _resolve_asset(raw: str, script_host: str, script_path: str):
+    """Resolve a reference found in a script to (host, templated path); None if not a file path."""
+    raw = raw.strip()
+    if not raw or raw.startswith(("data:", "blob:", "{param}")) or "{param}" in raw:
+        return None
+    try:
+        parts = urllib.parse.urlsplit(raw if "://" in raw or raw.startswith("//") else
+                                      urllib.parse.urljoin("https://x" + script_path, raw))
+    except ValueError:
+        return None
+    if "://" in raw or raw.startswith("//"):
+        host = (parts.hostname or "").lower() or script_host
+        path = parts.path
+    else:
+        host, path = script_host, parts.path
+    if not path.startswith("/") or "//" in path or len(path) > 200:
+        return None
+    return host, template_path(path)
+
+
+def extract_script_assets(body: str, host: str, path: str,
+                          header_map: str | None = None) -> set[tuple[str, str, str]]:
+    """Source maps and workers a script names: a set of (kind, host, path).
+
+    kind is "sourcemap", "worker", "service_worker" or "import_scripts". Like every code
+    reference these are claims about the code; the model records them as INFERRED."""
+    out: set = set()
+    refs = [("sourcemap", m.group(1)) for m in _SOURCEMAP_RE.finditer(body or "")]
+    if header_map:
+        refs.append(("sourcemap", header_map))
+    for kind, rx in _WORKER_RES:
+        refs += [(kind, m.group(2)) for m in rx.finditer(body or "")]
+    for kind, raw in refs:
+        r = _resolve_asset(raw, host, path)
+        if r:
+            out.add((kind, r[0], r[1]))
+    return out

proxy2model/ingest/infer.py

@@ -0,0 +1,75 @@
+"""
+Infer what kind of value a parameter carries, from its raw value.
+
+Runs in parse.py before values are masked, so only the label survives — the
+value itself never reaches the model. The checks run as a waterfall, most
+specific first, and the first match wins:
+
+    jwt -> uuid -> email -> integer -> number -> boolean -> base64 -> string
+
+An empty value says nothing about its type and yields None.
+"""
+
+from __future__ import annotations
+
+import re
+
+TYPES = ("jwt", "uuid", "email", "integer", "number", "boolean", "base64", "string")
+
+# ASCII classes only, so the Java engine (Infer.java) labels every value the same way
+_JWT_RE = re.compile(r"eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*")
+_UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
+_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
+_INT_RE = re.compile(r"-?[0-9]{1,19}")
+_NUM_RE = re.compile(r"-?[0-9]+\.[0-9]+(?:[eE][+-]?[0-9]+)?")
+_B64_RE = re.compile(r"[A-Za-z0-9+/]+")
+_B64URL_RE = re.compile(r"[A-Za-z0-9_-]+")
+_UPPER_RE = re.compile(r"[A-Z]")
+_LOWER_RE = re.compile(r"[a-z]")
+_B64_TELL_RE = re.compile(r"[0-9+/=_-]")
+
+
+def _is_base64(v: str) -> bool:
+    # short alphanumerics (`admin`, `true1`) are valid base64 by accident: demand
+    # length, well-formed padding, and the mix of cases and digits words rarely have
+    core = v.rstrip("=")
+    pad = len(v) - len(core)
+    if pad > 2 or (pad and len(v) % 4) or len(core) < 12 or len(core) % 4 == 1:
+        return False
+    if not (_B64_RE.fullmatch(core) or _B64URL_RE.fullmatch(core)):
+        return False
+    return bool(_B64_TELL_RE.search(v) and _UPPER_RE.search(v) and _LOWER_RE.search(v))
+
+
+def value_type(value: str | None) -> str | None:
+    """The type label for one raw value, or None when it is empty."""
+    if value is None:
+        return None
+    v = value.strip()
+    if not v:
+        return None
+    if _JWT_RE.fullmatch(v):
+        return "jwt"
+    if _UUID_RE.fullmatch(v):
+        return "uuid"
+    if _EMAIL_RE.fullmatch(v):
+        return "email"
+    if _INT_RE.fullmatch(v):
+        return "integer"
+    if _NUM_RE.fullmatch(v):
+        return "number"
+    if v.lower() in ("true", "false"):
+        return "boolean"
+    if _is_base64(v):
+        return "base64"
+    return "string"
+
+
+def param_types(location: str, params: list[tuple[str, str]]) -> list[tuple[str, str, str]]:
+    """(location, name, type) for each parameter whose value has a type."""
+    out = []
+    for name, value in params:
+        t = value_type(value)
+        if t is not None:
+            out.append((location, name, t))
+    return out

proxy2model/ingest/journey.py

@@ -0,0 +1,148 @@
+"""
+Scripted journeys: the flows an automatic crawl cannot invent.
+
+A crawler can click what is on the screen. It will not register an account, log
+in, fill a checkout, upload a file or ask for something a role should not reach.
+A journey is a short JSON file that says how, and the crawler performs it in the
+same browser session as the crawl, so the traffic lands in the same model (and
+an authenticated session carries on into the autonomous crawl that follows).
+
+    {
+      "name": "customer checkout",
+      "vars": {"email": "${env:SHOP_EMAIL}"},
+      "steps": [
+        {"request": {"method": "POST", "path": "/rest/user/login",
+                     "json": {"email": "${email}", "password": "${env:SHOP_PASSWORD}"},
+                     "extract": {"token": "authentication.token", "bid": "authentication.bid"}}},
+        {"set_header": {"Authorization": "Bearer ${token}"}},
+        {"set_storage": {"token": "${token}"}},
+        {"goto": "/#/basket"},
+        {"click": {"text": "Checkout"}},
+        {"fill": {"selector": "#address", "value": "1 Probe Street"}},
+        {"press": "Enter"},
+        {"wait_idle": true}
+      ]
+    }
+
+Steps:  goto  click  fill  press  wait  wait_for  wait_idle  request  set_header
+        set_storage  set_cookie  scroll  assert  (browser mode; a static crawl runs
+        goto, request, set_header and set_cookie only)
+
+`assert` checks where the journey has got to: {"assert": {"url_contains": "/welcome",
+"text": "Sign out", "not_text": "Invalid password", "selector": "#account"}}. A login
+that did not log in is the usual reason, and without it the crawl would carry on
+anonymously and say nothing.
+
+Values may use ${name} (a var, or something an earlier `extract` saved) and
+${env:NAME} (an environment variable: keep passwords out of the file and out of
+your shell history). Each step may carry "required": true to abort the journey if
+it fails; otherwise a failure is reported and the journey carries on.
+
+A journey is deliberate, so its requests are not held to the crawler's
+"looks destructive" filter. They are still held to the scope, to --read-only, and
+to --yes.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from dataclasses import dataclass, field
+
+STEPS = {"goto", "click", "fill", "press", "wait", "wait_for", "wait_idle", "request",
+         "set_header", "set_storage", "set_cookie", "scroll", "assert"}
+BROWSER_ONLY = {"click", "fill", "press", "wait_for", "wait_idle", "set_storage", "scroll", "assert"}
+_VAR = re.compile(r"\$\{(env:)?([A-Za-z_][A-Za-z0-9_]*)\}")
+
+
+class JourneyError(ValueError):
+    """A journey file that cannot be run (bad JSON, unknown step, missing variable)."""
+
+
+@dataclass
+class Journey:
+    name: str
+    vars: dict = field(default_factory=dict)
+    steps: list = field(default_factory=list)
+    path: str = ""
+
+
+def load(path: str) -> Journey:
+    try:
+        with open(path, encoding="utf-8") as f:
+            data = json.load(f)
+    except OSError as e:
+        raise JourneyError(f"cannot read journey {path}: {e}")
+    except ValueError as e:
+        raise JourneyError(f"{path} is not valid JSON: {e}")
+    if isinstance(data, list):
+        data = {"steps": data}
+    if not isinstance(data, dict) or not isinstance(data.get("steps"), list) or not data["steps"]:
+        raise JourneyError(f"{path}: a journey needs a non-empty \"steps\" list")
+    j = Journey(name=str(data.get("name") or os.path.basename(path)), vars=dict(data.get("vars") or {}),
+                steps=data["steps"], path=path)
+    for n, step in enumerate(j.steps, 1):
+        validate_step(step, f"{path} step {n}")
+    return j
+
+
+def validate_step(step, where: str) -> None:
+    if not isinstance(step, dict):
+        raise JourneyError(f"{where}: a step is an object like {{\"goto\": \"/\"}}")
+    kinds = [k for k in step if k in STEPS]
+    if len(kinds) != 1:
+        extra = [k for k in step if k not in STEPS and k not in ("required", "label")]
+        raise JourneyError(f"{where}: exactly one of {sorted(STEPS)} per step"
+                           + (f" (unknown key {extra[0]!r})" if extra else ""))
+    kind, arg = kinds[0], step[kinds[0]]
+    if kind == "goto" and not isinstance(arg, str):
+        raise JourneyError(f"{where}: goto takes a URL or path string")
+    if kind in ("click", "fill", "wait_for") and not isinstance(arg, (str, dict)):
+        raise JourneyError(f"{where}: {kind} takes a selector or {{\"text\": ...}}")
+    if kind == "fill" and not (isinstance(arg, dict) and "value" in arg and ("selector" in arg or "name" in arg)):
+        raise JourneyError(f"{where}: fill takes {{\"selector\": ..., \"value\": ...}}")
+    if kind == "request":
+        if not (isinstance(arg, dict) and arg.get("path") or isinstance(arg, dict) and arg.get("url")):
+            raise JourneyError(f"{where}: request takes {{\"method\": ..., \"path\": ...}}")
+    if kind == "assert" and not (isinstance(arg, dict) and set(arg) & {"url_contains", "text", "not_text", "selector"}):
+        raise JourneyError(f"{where}: assert takes url_contains, text, not_text or selector")
+    if kind in ("set_header", "set_storage", "set_cookie") and not isinstance(arg, dict):
+        raise JourneyError(f"{where}: {kind} takes an object of name: value")
+
+
+def substitute(value, variables: dict):
+    """Replace ${name} and ${env:NAME} anywhere in a step; a missing one is an error."""
+    if isinstance(value, str):
+        # a value that is exactly one ${var} keeps the variable's type, so an extracted
+        # id stays a JSON number instead of becoming "7"
+        whole = _VAR.fullmatch(value)
+        if whole and not whole.group(1) and whole.group(2) in variables:
+            return variables[whole.group(2)]
+
+        def one(m: re.Match) -> str:
+            name = m.group(2)
+            if m.group(1):
+                if name not in os.environ:
+                    raise JourneyError(f"environment variable {name} is not set (used as ${{env:{name}}})")
+                return os.environ[name]
+            if name not in variables:
+                raise JourneyError(f"variable ${{{name}}} is not defined (set it under \"vars\", "
+                                   "pass --var, or extract it in an earlier step)")
+            return str(variables[name])
+        return _VAR.sub(one, value)
+    if isinstance(value, dict):
+        return {k: substitute(v, variables) for k, v in value.items()}
+    if isinstance(value, list):
+        return [substitute(v, variables) for v in value]
+    return value
+
+
+def dig(node, path: str):
+    """`authentication.token`, `data.0.id` -> the value, or None."""
+    for part in path.split("."):
+        try:
+            node = node[int(part)] if isinstance(node, list) else node[part]
+        except (KeyError, IndexError, ValueError, TypeError):
+            return None
+    return node

proxy2model/ingest/live.py

@@ -0,0 +1,96 @@
+"""
+Live capture: exchanges recorded while you browse are masked as they arrive, appended to
+`inputs/browse.jsonl`, and the model is rebuilt every few seconds while anything new comes in.
+
+Only the masked record is ever written; a restart continues where the last session stopped.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import threading
+import time
+from xml.etree.ElementTree import Element
+
+from .parse import Exchange, parse_elements
+
+
+class Live:
+    """Masked exchanges waiting to be saved, and the file they are appended to."""
+
+    def __init__(self, path: str, role: str | None = None):
+        self.path = path
+        self.role = role
+        self.lock = threading.Lock()
+        self.pending: list[Exchange] = []
+        self.stats = {"items": 0, "parsed": 0, "skipped": 0}
+        self.received = 0
+        self.saved = 0
+        if os.path.exists(path):
+            with open(path, encoding="utf-8") as f:
+                self.saved = sum(1 for line in f if line.strip() and '"_stats"' not in line[:12])
+
+    def ingest(self, items: list[Element]) -> tuple[int, int]:
+        """Parse and mask each recorded <item>; returns (accepted, skipped)."""
+        stats: dict = {}
+        parsed = list(parse_elements(items, source="live", role=self.role, stats=stats))
+        skipped = len(items) - len(parsed)
+        with self.lock:
+            for ex in parsed:
+                ex.item = self.saved + len(self.pending)
+                self.pending.append(ex)
+            self.stats["items"] += len(items)
+            self.stats["parsed"] += len(parsed)
+            self.stats["skipped"] += skipped
+            self.received += len(parsed)
+        return len(parsed), skipped
+
+    def flush(self) -> int:
+        """Append what is pending to the input file in one write; returns how many were saved."""
+        with self.lock:
+            batch, stats = self.pending, self.stats
+            self.pending, self.stats = [], {"items": 0, "parsed": 0, "skipped": 0}
+        if not batch and not stats["items"]:
+            return 0
+        lines = [json.dumps(ex.to_record()) for ex in batch]
+        lines.append(json.dumps({"_stats": stats, "_version": 1}))
+        os.makedirs(os.path.dirname(self.path), exist_ok=True)
+        with open(self.path, "a", encoding="utf-8") as f:
+            f.write("\n".join(lines) + "\n")
+            f.flush()
+            os.fsync(f.fileno())
+        self.saved += len(batch)
+        return len(batch)
+
+
+class Rebuilder(threading.Thread):
+    """Every `interval` seconds, saves what arrived and, if anything did, rebuilds.
+
+    `report` gets one dict per rebuild: {"added", "saved", "summary", "seconds"}, or {"error"}."""
+
+    def __init__(self, live: Live, rebuild, interval: float, report=print):
+        super().__init__(daemon=True)
+        self.live, self.rebuild, self.interval, self.report = live, rebuild, interval, report
+        self.stop = threading.Event()
+        self.last = 0.0                              # how long the last rebuild took
+
+    def step(self) -> None:
+        n = self.live.flush()
+        if not n:
+            return
+        t0 = time.monotonic()
+        try:
+            summary = self.rebuild()
+        except Exception as e:                      # a bad rebuild must not stop the listener
+            self.report({"error": str(e)})
+        else:
+            self.report({"added": n, "saved": self.live.saved, "summary": summary,
+                         "seconds": time.monotonic() - t0})
+        self.last = time.monotonic() - t0
+
+    def run(self) -> None:
+        # a big session rebuilds slowly: wait at least twice as long as the last rebuild took,
+        # so the listener never spends more than a third of its time rebuilding
+        while not self.stop.wait(max(self.interval, 2 * self.last)):
+            self.step()