rea-agents 5.0.0 → 6.0.0

package/bridge/mitmproxy/capture.py

@@ -0,0 +1,210 @@
+"""REA offline adapter using mitmproxy's unchanged native tnetstring decoder.
+
+No FlowReader migrations, HAR conversion, proxy listener, or target requests.
+The containing mitmdump process and its configuration directory are REA-owned.
+"""
+
+import base64
+import hashlib
+import json
+import math
+import re
+import resource
+from pathlib import Path
+
+from mitmproxy import ctx, version
+from mitmproxy.io import tnetstring
+
+PROFILE = "12.2.3"
+OUTPUT_BYTES = 96 * 1024 * 1024
+CREDENTIALS = {b"authorization", b"proxy-authorization", b"cookie", b"set-cookie"}
+
+
+class CaptureFailure(Exception):
+    def __init__(self, reason, message, pointer=""):
+        self.reason, self.pointer = reason, pointer
+        super().__init__(message)
+
+
+def pointer(parent, key):
+    return parent + "/" + str(key).replace("~", "~0").replace("/", "~1")
+
+
+class BoundedReader:
+    """Refuse malicious declared lengths before the upstream parser allocates."""
+    def __init__(self, handle, size):
+        self.handle, self.size = handle, size
+
+    def read(self, count):
+        if count < 0 or count > self.size - self.handle.tell():
+            raise CaptureFailure("format", "Native record declares bytes beyond the retained capture.")
+        return self.handle.read(count)
+
+
+def project_record(state, secrets):
+    binaries, numbers, redactions = [], [], []
+    byte_secrets = []
+    for secret in secrets:
+        try:
+            byte_secrets.append(secret.encode("utf-8", errors="strict"))
+        except UnicodeEncodeError:
+            # Non-scalar JSON strings have no UTF-8 byte representation.
+            # Keep them in secrets for the ordinary string projection.
+            pass
+
+    def redact(text, path, original):
+        if any(secret in text or secret in original for secret in secrets):
+            redactions.append({"pointer": path, "reason": "explicit-sensitive-value"})
+            return None
+        return text
+
+    def transport_value(value, path, transport_url):
+        if not (transport_url or re.fullmatch(r"/(?:backup/)*request/(?:path|authority)", path)):
+            return value, False
+        binary = isinstance(value, bytes)
+        pattern = rb"^([\x00-\x20]*(?:[a-z][a-z0-9+.-]*:)?//)[^/?#]*@" if binary else r"^([\x00-\x20]*(?:[a-z][a-z0-9+.-]*:)?//)[^/?#]*@"
+        replacement = rb"\1" if binary else r"\1"
+        separator = b"@" if binary else "@"
+        safe_url = re.sub(pattern, replacement, value, flags=re.I)
+        if path.endswith("/authority") and separator in safe_url:
+            safe_url = safe_url.rsplit(separator, 1)[1]
+        changed = safe_url != value
+        if changed:
+            redactions.append({"pointer": path, "reason": "transport-credential"})
+        return safe_url, changed
+
+    def validate_omitted(value, path, depth):
+        before = len(binaries), len(numbers), len(redactions)
+        visit(value, path, depth)
+        del binaries[before[0]:]
+        del numbers[before[1]:]
+        del redactions[before[2]:]
+
+    def visit(value, path, depth=0, credential=False, transport_url=False):
+        if depth > 64:
+            raise CaptureFailure("input-limit", "Native capture exceeds the 64-level evidence nesting budget.", path)
+        if credential:
+            validate_omitted(value, path, depth)
+            redactions.append({"pointer": path, "reason": "transport-credential"})
+            if isinstance(value, bytes):
+                binaries.append({"pointer": path, "representation": "producer-bytes", "state": "redacted", "content_base64": None, "bytes": None, "sha256": None})
+            return None
+        if isinstance(value, bytes):
+            safe_bytes, credential_url = transport_value(value, path, transport_url)
+            content_base64 = base64.b64encode(value).decode("ascii")
+            digest = hashlib.sha256(value).hexdigest()
+            hidden = any(secret in value or secret in safe_bytes for secret in byte_secrets) or any(secret in content_base64 or secret in digest for secret in secrets)
+            safe_text = None
+            try:
+                safe_text = safe_bytes.decode("utf-8", errors="strict")
+            except UnicodeDecodeError:
+                pass
+            if hidden:
+                redactions.append({"pointer": path, "reason": "explicit-sensitive-value"})
+            exclude = hidden or credential_url
+            binaries.append({"pointer": path, "representation": "producer-bytes", "state": "redacted" if exclude else "retained", "content_base64": None if exclude else content_base64, "bytes": None if exclude else len(value), "sha256": None if exclude else digest})
+            if hidden:
+                return None
+            return safe_text
+        if isinstance(value, str):
+            safe_text, _ = transport_value(value, path, transport_url)
+            return redact(safe_text, path, value)
+        if isinstance(value, bool) or value is None:
+            return value
+        if isinstance(value, (int, float)):
+            literal = repr(value)
+            if any(secret in literal for secret in secrets):
+                redactions.append({"pointer": path, "reason": "explicit-sensitive-value"})
+                return None
+            numbers.append({"pointer": path, "producer_type": "integer" if isinstance(value, int) else "float", "literal": literal})
+            return value if (abs(value) <= 9007199254740991 if isinstance(value, int) else math.isfinite(value)) else None
+        if isinstance(value, (list, tuple)):
+            fields = re.fullmatch(r"/(?:backup/)*(?:request|response)/(?:headers|trailers)", path)
+            if fields:
+                result = []
+                for index, field in enumerate(value):
+                    if not isinstance(field, (list, tuple)) or len(field) != 2 or not isinstance(field[0], bytes):
+                        raise CaptureFailure("format", "Native header/trailer is not an ordered byte pair.", pointer(path, index))
+                    result.append([visit(field[0], pointer(pointer(path, index), 0), depth + 2), visit(field[1], pointer(pointer(path, index), 1), depth + 2, field[0].lower() in CREDENTIALS, field[0].lower() in {b"location", b"referer", b"origin"})])
+                return result
+            return [visit(item, pointer(path, index), depth + 1) for index, item in enumerate(value)]
+        if isinstance(value, dict):
+            result = {}
+            for key, item in value.items():
+                if not isinstance(key, str):
+                    raise CaptureFailure("format", "Native record has a non-string dictionary key.", path)
+                if key == "__proto__":
+                    raise CaptureFailure("unsupported", "Native object contains a __proto__ member that the current JSON schema boundary cannot preserve.", path)
+                if any(secret in key for secret in secrets):
+                    # Validate the omitted subtree too, then discard all of its
+                    # coordinates rather than inventing a replacement identity.
+                    validate_omitted(item, pointer(path, key), depth + 1)
+                    redactions.append({"pointer": path, "reason": "explicit-sensitive-value", "scope": "property-name"})
+                    continue
+                result[key] = visit(item, pointer(path, key), depth + 1)
+            return result
+        raise CaptureFailure("decoder", "Native parser returned an unexpected value type: " + type(value).__name__ + ".", path)
+
+    return {"reported": visit(state, ""), "binary_fields": binaries, "numeric_literals": numbers, "redactions": redactions}
+
+
+def decode(request):
+    if version.VERSION != PROFILE:
+        raise CaptureFailure("unsupported", "Expected mitmproxy " + PROFILE + "; observed " + version.VERSION + ".")
+    records = []
+    capture = Path(request["snapshot_path"])
+    size = capture.stat().st_size
+    with capture.open("rb") as handle:
+        reader = BoundedReader(handle, size)
+        while handle.tell() < size:
+            start = handle.tell()
+            try:
+                state = tnetstring.load(reader)
+            except CaptureFailure:
+                raise
+            except RecursionError:
+                raise CaptureFailure("input-limit", "Native parser exceeded its recursion budget at byte offset " + str(start) + ".") from None
+            except (ValueError, IndexError, UnicodeError, EOFError):
+                raise CaptureFailure("format", "Malformed native record at byte offset " + str(start) + ".") from None
+            if not isinstance(state, dict):
+                raise CaptureFailure("format", "Native flow record must be a dictionary at byte offset " + str(start) + ".")
+            records.append({"ordinal": len(records), "location": {"kind": "byte-range", "offset": start, "bytes": handle.tell() - start}, **project_record(state, request["sensitive_values"]), "limitations": ["Fields are original native states without FlowReader migration. Binary fields contain exact producer bytes; reported UTF-8 strings for those fields are derived display views. Unknown flow/version extensions are retained without interpretation."]})
+    return {"decoder": {"id": "mitmproxy-native-tnetstring", "name": "REA offline native mitmproxy adapter", "version": PROFILE}, "container": {"reported": None, "numeric_literals": [], "redactions": [], "records_pointer": None}, "total_records": len(records), "records": records}
+
+
+def failure_reply(error):
+    """Keep producer constraints, resource exhaustion and adapter defects distinct."""
+    if isinstance(error, CaptureFailure):
+        return {"ok": False, "reason": error.reason, "message": str(error), "pointer": error.pointer}
+    if isinstance(error, MemoryError):
+        return {"ok": False, "reason": "resource-limit", "message": "Native decoder exhausted its 768 MiB address-space budget.", "pointer": ""}
+    return {"ok": False, "reason": "decoder", "message": "Offline native decoder failed unexpectedly: " + type(error).__name__ + ".", "pointer": ""}
+
+
+class OfflineCapture:
+    def load(self, loader):
+        loader.add_option("rea_request_path", str, "", "REA private offline capture request")
+
+    def running(self):
+        resource.setrlimit(resource.RLIMIT_AS, (768 * 1024 * 1024, 768 * 1024 * 1024))
+        resource.setrlimit(resource.RLIMIT_CPU, (30, 30))
+        resource.setrlimit(resource.RLIMIT_FSIZE, (OUTPUT_BYTES, OUTPUT_BYTES))
+        resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
+        request = json.loads(Path(ctx.options.rea_request_path).read_text(encoding="utf-8"))
+        try:
+            response = {"ok": True, "value": decode(request)}
+        except Exception as error:
+            response = failure_reply(error)
+        try:
+            try:
+                output = json.dumps(response, ensure_ascii=True, allow_nan=False, separators=(",", ":")).encode("ascii")
+            except Exception as error:
+                output = json.dumps(failure_reply(error), separators=(",", ":")).encode("ascii")
+            if len(output) > OUTPUT_BYTES:
+                output = b'{"ok":false,"reason":"limit","message":"Native reply exceeds the complete-evidence output budget.","pointer":""}'
+            Path(request["reply_path"]).write_bytes(output)
+        finally:
+            ctx.master.shutdown()
+
+
+addons = [OfflineCapture()]

package/bridge/native/rea_lldb_tracer.py

@@ -0,0 +1,303 @@
+"""Bounded native call observation executed inside LLDB's embedded Python.
+
+REA imports this module into `lldb --batch` and runs `rea_trace <config.json>`.
+It launches one owned process, stops at the requested functions and
+Objective-C methods, records each entry from registers and memory reads only,
+continues, and kills the process once it exits, the event limit is reached or
+the duration elapses. No expression is evaluated and no target code runs on
+the tracer's behalf. The result is written as JSON to the configured path.
+"""
+
+import json
+import time
+
+import lldb
+
+ARGUMENT_REGISTERS = {
+    "arm64": ["x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7"],
+    "arm64e": ["x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7"],
+    "x86_64": ["rdi", "rsi", "rdx", "rcx", "r8", "r9"],
+    "x86_64h": ["rdi", "rsi", "rdx", "rcx", "r8", "r9"],
+}
+MAX_SELECTOR_BYTES = 1024
+MAX_REPORTED_LOCATIONS = 64
+MAX_OTHER_STOPS = 256
+LISTENER_NAME = "rea-native-call-observation"
+
+
+def _hex(value):
+    return "0x%x" % value
+
+
+def _location(address, target):
+    """Load and file address, module and symbol of one code address."""
+    module = address.GetModule()
+    symbol = address.GetSymbol()
+    file_address = address.GetFileAddress()
+    return {
+        "load_address": _hex(address.GetLoadAddress(target)),
+        "file_address": None
+        if file_address == lldb.LLDB_INVALID_ADDRESS
+        else _hex(file_address),
+        "module": module.GetFileSpec().GetFilename() if module.IsValid() else None,
+        "module_path": str(module.GetFileSpec()) if module.IsValid() else None,
+        "symbol": symbol.GetName() if symbol.IsValid() else None,
+    }
+
+
+def _no_files():
+    return lldb.SBFileSpecList()
+
+
+def _create_breakpoints(target, spec):
+    """LLDB breakpoints for one request; a class-qualified method may need two."""
+    if spec["kind"] == "function":
+        modules = lldb.SBFileSpecList()
+        if spec.get("module"):
+            modules.Append(lldb.SBFileSpec(spec["module"], False))
+        return [
+            target.BreakpointCreateByName(
+                spec["name"], lldb.eFunctionNameTypeAuto, modules, _no_files()
+            )
+        ]
+    class_name = spec.get("class_name")
+    method_type = spec.get("method_type", "any")
+    if class_name is None:
+        return [
+            target.BreakpointCreateByName(
+                spec["selector"], lldb.eFunctionNameTypeSelector, _no_files(), _no_files()
+            )
+        ]
+    prefixes = {"instance": ["-"], "class": ["+"], "any": ["-", "+"]}[method_type]
+    return [
+        target.BreakpointCreateByName(
+            "%s[%s %s]" % (prefix, class_name, spec["selector"]),
+            lldb.eFunctionNameTypeFull,
+            _no_files(),
+            _no_files(),
+        )
+        for prefix in prefixes
+    ]
+
+
+def _dynamic_class(target, pointer):
+    """Class of an Objective-C receiver from LLDB's runtime reader, without running code."""
+    if pointer == 0:
+        return None
+    data = lldb.SBData.CreateDataFromUInt64Array(
+        target.GetByteOrder(), target.GetAddressByteSize(), [pointer]
+    )
+    value = target.CreateValueFromData(
+        "receiver", data, target.GetBasicType(lldb.eBasicTypeObjCID)
+    )
+    dynamic = value.GetDynamicValue(lldb.eDynamicDontRunTarget)
+    name = dynamic.GetTypeName() if dynamic.IsValid() else None
+    if not name or name in ("id", "objc_object *"):
+        return None
+    return name[:-2] if name.endswith(" *") else name
+
+
+def _wanted(spec, symbol):
+    """Selector-only requests may admit only instance or only class methods."""
+    if spec["kind"] != "objc-method" or spec.get("class_name") is not None:
+        return True
+    method_type = spec.get("method_type", "any")
+    if method_type == "any":
+        return True
+    return symbol.startswith("-[" if method_type == "instance" else "+[")
+
+
+def _record(context, thread, index, sequence):
+    target, process, config = context["target"], context["process"], context["config"]
+    frame = thread.GetFrameAtIndex(0)
+    location = _location(frame.GetPCAddress(), target)
+    symbol = location["symbol"] or ""
+    if not _wanted(config["breakpoints"][index], symbol):
+        return None
+    names = ARGUMENT_REGISTERS.get(config["architecture"], [])
+    registers = [
+        {"name": name, "value": _hex(frame.FindRegister(name).GetValueAsUnsigned())}
+        for name in names[: config["argument_registers"]]
+    ]
+    receiver_class = None
+    selector = None
+    if symbol.startswith("-[") or symbol.startswith("+["):
+        error = lldb.SBError()
+        text = process.ReadCStringFromMemory(
+            frame.FindRegister(names[1]).GetValueAsUnsigned(),
+            MAX_SELECTOR_BYTES,
+            error,
+        )
+        selector = text if error.Success() and text else None
+        if symbol.startswith("-["):
+            receiver_class = _dynamic_class(
+                target, frame.FindRegister(names[0]).GetValueAsUnsigned()
+            )
+    frames = min(thread.GetNumFrames(), 1 + config["backtrace_frames"])
+    record = {
+        "sequence": sequence,
+        "elapsed_ms": round((time.monotonic() - context["started"]) * 1000, 3),
+        "thread_id": thread.GetThreadID(),
+        "breakpoint_index": index,
+        "receiver_class": receiver_class,
+        "selector": selector,
+        "registers": registers,
+        "backtrace": [
+            _location(thread.GetFrameAtIndex(depth).GetPCAddress(), target)
+            for depth in range(1, frames)
+        ],
+    }
+    record.update(location)
+    return record
+
+
+def _handle_stop(context, events, other_stops):
+    """Record breakpoint hits of one stop; return an outcome when tracing must end."""
+    process = context["process"]
+    for thread in process:
+        reason = thread.GetStopReason()
+        if reason == lldb.eStopReasonBreakpoint:
+            index = context["breakpoint_index"].get(thread.GetStopReasonDataAtIndex(0))
+            if index is None:
+                continue
+            record = _record(context, thread, index, len(events))
+            if record is not None:
+                events.append(record)
+            if len(events) >= context["config"]["max_events"]:
+                return "event-limit"
+        elif reason in (lldb.eStopReasonSignal, lldb.eStopReasonException):
+            other_stops.append(thread.GetStopDescription(256))
+            if len(other_stops) >= MAX_OTHER_STOPS:
+                return "stop-limit"
+    return None
+
+
+def _breakpoint_report(target, created):
+    report = []
+    for index, breakpoints in enumerate(created):
+        locations = []
+        count = 0
+        for breakpoint in breakpoints:
+            count += breakpoint.GetNumLocations()
+            for position in range(breakpoint.GetNumLocations()):
+                if len(locations) < MAX_REPORTED_LOCATIONS:
+                    locations.append(
+                        _location(breakpoint.GetLocationAtIndex(position).GetAddress(), target)
+                    )
+        report.append({"index": index, "location_count": count, "locations": locations})
+    return report
+
+
+def _launch_info(target, config, listener):
+    launch = lldb.SBLaunchInfo(config["arguments"])
+    launch.SetListener(listener)
+    environment = target.GetEnvironment()
+    for key, value in config["environment"].items():
+        environment.Set(key, value, True)
+    launch.SetEnvironment(environment, False)
+    if config.get("working_directory"):
+        launch.SetWorkingDirectory(config["working_directory"])
+    launch.AddOpenFileAction(0, "/dev/null", True, False)
+    launch.AddOpenFileAction(1, config["stdout_path"], False, True)
+    launch.AddOpenFileAction(2, config["stderr_path"], False, True)
+    return launch
+
+
+def _wait_for_exit(listener, process, seconds):
+    event = lldb.SBEvent()
+    deadline = time.monotonic() + seconds
+    while process.GetState() != lldb.eStateExited and time.monotonic() < deadline:
+        listener.WaitForEvent(1, event)
+
+
+def trace(debugger, config):
+    debugger.SetAsync(True)
+    # Stop at the symbol itself, where argument registers still hold the ABI
+    # arguments, even when line tables would let LLDB skip the prologue.
+    lldb.SBDebugger.SetInternalVariable(
+        "target.skip-prologue", "false", debugger.GetInstanceName()
+    )
+    error = lldb.SBError()
+    target = debugger.CreateTarget(
+        config["executable"], config["architecture"], None, False, error
+    )
+    if not target.IsValid():
+        return {"status": "target-error", "error": error.GetCString()}
+    created = [_create_breakpoints(target, spec) for spec in config["breakpoints"]]
+    breakpoint_index = {}
+    for index, breakpoints in enumerate(created):
+        for breakpoint in breakpoints:
+            breakpoint_index[breakpoint.GetID()] = index
+    listener = lldb.SBListener(LISTENER_NAME)
+    started = time.monotonic()
+    process = target.Launch(_launch_info(target, config, listener), error)
+    if not error.Success() or not process.IsValid():
+        return {"status": "launch-error", "error": error.GetCString()}
+    with open(config["pid_path"], "w") as handle:
+        handle.write(str(process.GetProcessID()))
+    context = {
+        "target": target,
+        "process": process,
+        "config": config,
+        "started": started,
+        "breakpoint_index": breakpoint_index,
+    }
+    events = []
+    other_stops = []
+    outcome = None
+    deadline = started + config["duration_ms"] / 1000
+    event = lldb.SBEvent()
+    while outcome is None:
+        if time.monotonic() >= deadline:
+            outcome = "duration-elapsed"
+            break
+        if not listener.WaitForEvent(1, event):
+            continue
+        if not lldb.SBProcess.EventIsProcessEvent(event):
+            continue
+        state = lldb.SBProcess.GetStateFromEvent(event)
+        if state == lldb.eStateExited:
+            outcome = "exited"
+        elif state == lldb.eStateStopped and not lldb.SBProcess.GetRestartedFromEvent(event):
+            outcome = _handle_stop(context, events, other_stops)
+            if outcome is None:
+                process.Continue()
+    pid = process.GetProcessID()
+    exit_status = None
+    exit_description = None
+    if outcome == "exited":
+        exit_status = process.GetExitStatus()
+        exit_description = process.GetExitDescription()
+    else:
+        process.Kill()
+        _wait_for_exit(listener, process, 5)
+    return {
+        "status": "traced",
+        "version": lldb.SBDebugger.GetVersionString().splitlines()[0],
+        "pid": pid,
+        "outcome": outcome,
+        "exit_status": exit_status,
+        "exit_description": exit_description,
+        "killed": outcome != "exited",
+        "terminated": process.GetState() == lldb.eStateExited,
+        "elapsed_ms": round((time.monotonic() - started) * 1000, 3),
+        "breakpoints": _breakpoint_report(target, created),
+        "events": events,
+        "other_stops": other_stops,
+    }
+
+
+def run(debugger, command, result, internal_dict):
+    """LLDB command entry point: `rea_trace <config.json>`."""
+    with open(command.strip()) as handle:
+        config = json.load(handle)
+    try:
+        output = trace(debugger, config)
+    except Exception as error:  # Reported as a tracer failure, never re-raised into LLDB.
+        output = {"status": "tracer-error", "error": "%s: %s" % (type(error).__name__, error)}
+    with open(config["result_path"], "w") as handle:
+        json.dump(output, handle)
+
+
+def __lldb_init_module(debugger, internal_dict):
+    debugger.HandleCommand("command script add -f rea_lldb_tracer.run rea_trace")