rea-agents 4.1.0 → 6.0.0

package/bridge/native/rea_lldb_tracer.py

@@ -0,0 +1,303 @@
+"""Bounded native call observation executed inside LLDB's embedded Python.
+
+REA imports this module into `lldb --batch` and runs `rea_trace <config.json>`.
+It launches one owned process, stops at the requested functions and
+Objective-C methods, records each entry from registers and memory reads only,
+continues, and kills the process once it exits, the event limit is reached or
+the duration elapses. No expression is evaluated and no target code runs on
+the tracer's behalf. The result is written as JSON to the configured path.
+"""
+
+import json
+import time
+
+import lldb
+
+ARGUMENT_REGISTERS = {
+    "arm64": ["x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7"],
+    "arm64e": ["x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7"],
+    "x86_64": ["rdi", "rsi", "rdx", "rcx", "r8", "r9"],
+    "x86_64h": ["rdi", "rsi", "rdx", "rcx", "r8", "r9"],
+}
+MAX_SELECTOR_BYTES = 1024
+MAX_REPORTED_LOCATIONS = 64
+MAX_OTHER_STOPS = 256
+LISTENER_NAME = "rea-native-call-observation"
+
+
+def _hex(value):
+    return "0x%x" % value
+
+
+def _location(address, target):
+    """Load and file address, module and symbol of one code address."""
+    module = address.GetModule()
+    symbol = address.GetSymbol()
+    file_address = address.GetFileAddress()
+    return {
+        "load_address": _hex(address.GetLoadAddress(target)),
+        "file_address": None
+        if file_address == lldb.LLDB_INVALID_ADDRESS
+        else _hex(file_address),
+        "module": module.GetFileSpec().GetFilename() if module.IsValid() else None,
+        "module_path": str(module.GetFileSpec()) if module.IsValid() else None,
+        "symbol": symbol.GetName() if symbol.IsValid() else None,
+    }
+
+
+def _no_files():
+    return lldb.SBFileSpecList()
+
+
+def _create_breakpoints(target, spec):
+    """LLDB breakpoints for one request; a class-qualified method may need two."""
+    if spec["kind"] == "function":
+        modules = lldb.SBFileSpecList()
+        if spec.get("module"):
+            modules.Append(lldb.SBFileSpec(spec["module"], False))
+        return [
+            target.BreakpointCreateByName(
+                spec["name"], lldb.eFunctionNameTypeAuto, modules, _no_files()
+            )
+        ]
+    class_name = spec.get("class_name")
+    method_type = spec.get("method_type", "any")
+    if class_name is None:
+        return [
+            target.BreakpointCreateByName(
+                spec["selector"], lldb.eFunctionNameTypeSelector, _no_files(), _no_files()
+            )
+        ]
+    prefixes = {"instance": ["-"], "class": ["+"], "any": ["-", "+"]}[method_type]
+    return [
+        target.BreakpointCreateByName(
+            "%s[%s %s]" % (prefix, class_name, spec["selector"]),
+            lldb.eFunctionNameTypeFull,
+            _no_files(),
+            _no_files(),
+        )
+        for prefix in prefixes
+    ]
+
+
+def _dynamic_class(target, pointer):
+    """Class of an Objective-C receiver from LLDB's runtime reader, without running code."""
+    if pointer == 0:
+        return None
+    data = lldb.SBData.CreateDataFromUInt64Array(
+        target.GetByteOrder(), target.GetAddressByteSize(), [pointer]
+    )
+    value = target.CreateValueFromData(
+        "receiver", data, target.GetBasicType(lldb.eBasicTypeObjCID)
+    )
+    dynamic = value.GetDynamicValue(lldb.eDynamicDontRunTarget)
+    name = dynamic.GetTypeName() if dynamic.IsValid() else None
+    if not name or name in ("id", "objc_object *"):
+        return None
+    return name[:-2] if name.endswith(" *") else name
+
+
+def _wanted(spec, symbol):
+    """Selector-only requests may admit only instance or only class methods."""
+    if spec["kind"] != "objc-method" or spec.get("class_name") is not None:
+        return True
+    method_type = spec.get("method_type", "any")
+    if method_type == "any":
+        return True
+    return symbol.startswith("-[" if method_type == "instance" else "+[")
+
+
+def _record(context, thread, index, sequence):
+    target, process, config = context["target"], context["process"], context["config"]
+    frame = thread.GetFrameAtIndex(0)
+    location = _location(frame.GetPCAddress(), target)
+    symbol = location["symbol"] or ""
+    if not _wanted(config["breakpoints"][index], symbol):
+        return None
+    names = ARGUMENT_REGISTERS.get(config["architecture"], [])
+    registers = [
+        {"name": name, "value": _hex(frame.FindRegister(name).GetValueAsUnsigned())}
+        for name in names[: config["argument_registers"]]
+    ]
+    receiver_class = None
+    selector = None
+    if symbol.startswith("-[") or symbol.startswith("+["):
+        error = lldb.SBError()
+        text = process.ReadCStringFromMemory(
+            frame.FindRegister(names[1]).GetValueAsUnsigned(),
+            MAX_SELECTOR_BYTES,
+            error,
+        )
+        selector = text if error.Success() and text else None
+        if symbol.startswith("-["):
+            receiver_class = _dynamic_class(
+                target, frame.FindRegister(names[0]).GetValueAsUnsigned()
+            )
+    frames = min(thread.GetNumFrames(), 1 + config["backtrace_frames"])
+    record = {
+        "sequence": sequence,
+        "elapsed_ms": round((time.monotonic() - context["started"]) * 1000, 3),
+        "thread_id": thread.GetThreadID(),
+        "breakpoint_index": index,
+        "receiver_class": receiver_class,
+        "selector": selector,
+        "registers": registers,
+        "backtrace": [
+            _location(thread.GetFrameAtIndex(depth).GetPCAddress(), target)
+            for depth in range(1, frames)
+        ],
+    }
+    record.update(location)
+    return record
+
+
+def _handle_stop(context, events, other_stops):
+    """Record breakpoint hits of one stop; return an outcome when tracing must end."""
+    process = context["process"]
+    for thread in process:
+        reason = thread.GetStopReason()
+        if reason == lldb.eStopReasonBreakpoint:
+            index = context["breakpoint_index"].get(thread.GetStopReasonDataAtIndex(0))
+            if index is None:
+                continue
+            record = _record(context, thread, index, len(events))
+            if record is not None:
+                events.append(record)
+            if len(events) >= context["config"]["max_events"]:
+                return "event-limit"
+        elif reason in (lldb.eStopReasonSignal, lldb.eStopReasonException):
+            other_stops.append(thread.GetStopDescription(256))
+            if len(other_stops) >= MAX_OTHER_STOPS:
+                return "stop-limit"
+    return None
+
+
+def _breakpoint_report(target, created):
+    report = []
+    for index, breakpoints in enumerate(created):
+        locations = []
+        count = 0
+        for breakpoint in breakpoints:
+            count += breakpoint.GetNumLocations()
+            for position in range(breakpoint.GetNumLocations()):
+                if len(locations) < MAX_REPORTED_LOCATIONS:
+                    locations.append(
+                        _location(breakpoint.GetLocationAtIndex(position).GetAddress(), target)
+                    )
+        report.append({"index": index, "location_count": count, "locations": locations})
+    return report
+
+
+def _launch_info(target, config, listener):
+    launch = lldb.SBLaunchInfo(config["arguments"])
+    launch.SetListener(listener)
+    environment = target.GetEnvironment()
+    for key, value in config["environment"].items():
+        environment.Set(key, value, True)
+    launch.SetEnvironment(environment, False)
+    if config.get("working_directory"):
+        launch.SetWorkingDirectory(config["working_directory"])
+    launch.AddOpenFileAction(0, "/dev/null", True, False)
+    launch.AddOpenFileAction(1, config["stdout_path"], False, True)
+    launch.AddOpenFileAction(2, config["stderr_path"], False, True)
+    return launch
+
+
+def _wait_for_exit(listener, process, seconds):
+    event = lldb.SBEvent()
+    deadline = time.monotonic() + seconds
+    while process.GetState() != lldb.eStateExited and time.monotonic() < deadline:
+        listener.WaitForEvent(1, event)
+
+
+def trace(debugger, config):
+    debugger.SetAsync(True)
+    # Stop at the symbol itself, where argument registers still hold the ABI
+    # arguments, even when line tables would let LLDB skip the prologue.
+    lldb.SBDebugger.SetInternalVariable(
+        "target.skip-prologue", "false", debugger.GetInstanceName()
+    )
+    error = lldb.SBError()
+    target = debugger.CreateTarget(
+        config["executable"], config["architecture"], None, False, error
+    )
+    if not target.IsValid():
+        return {"status": "target-error", "error": error.GetCString()}
+    created = [_create_breakpoints(target, spec) for spec in config["breakpoints"]]
+    breakpoint_index = {}
+    for index, breakpoints in enumerate(created):
+        for breakpoint in breakpoints:
+            breakpoint_index[breakpoint.GetID()] = index
+    listener = lldb.SBListener(LISTENER_NAME)
+    started = time.monotonic()
+    process = target.Launch(_launch_info(target, config, listener), error)
+    if not error.Success() or not process.IsValid():
+        return {"status": "launch-error", "error": error.GetCString()}
+    with open(config["pid_path"], "w") as handle:
+        handle.write(str(process.GetProcessID()))
+    context = {
+        "target": target,
+        "process": process,
+        "config": config,
+        "started": started,
+        "breakpoint_index": breakpoint_index,
+    }
+    events = []
+    other_stops = []
+    outcome = None
+    deadline = started + config["duration_ms"] / 1000
+    event = lldb.SBEvent()
+    while outcome is None:
+        if time.monotonic() >= deadline:
+            outcome = "duration-elapsed"
+            break
+        if not listener.WaitForEvent(1, event):
+            continue
+        if not lldb.SBProcess.EventIsProcessEvent(event):
+            continue
+        state = lldb.SBProcess.GetStateFromEvent(event)
+        if state == lldb.eStateExited:
+            outcome = "exited"
+        elif state == lldb.eStateStopped and not lldb.SBProcess.GetRestartedFromEvent(event):
+            outcome = _handle_stop(context, events, other_stops)
+            if outcome is None:
+                process.Continue()
+    pid = process.GetProcessID()
+    exit_status = None
+    exit_description = None
+    if outcome == "exited":
+        exit_status = process.GetExitStatus()
+        exit_description = process.GetExitDescription()
+    else:
+        process.Kill()
+        _wait_for_exit(listener, process, 5)
+    return {
+        "status": "traced",
+        "version": lldb.SBDebugger.GetVersionString().splitlines()[0],
+        "pid": pid,
+        "outcome": outcome,
+        "exit_status": exit_status,
+        "exit_description": exit_description,
+        "killed": outcome != "exited",
+        "terminated": process.GetState() == lldb.eStateExited,
+        "elapsed_ms": round((time.monotonic() - started) * 1000, 3),
+        "breakpoints": _breakpoint_report(target, created),
+        "events": events,
+        "other_stops": other_stops,
+    }
+
+
+def run(debugger, command, result, internal_dict):
+    """LLDB command entry point: `rea_trace <config.json>`."""
+    with open(command.strip()) as handle:
+        config = json.load(handle)
+    try:
+        output = trace(debugger, config)
+    except Exception as error:  # Reported as a tracer failure, never re-raised into LLDB.
+        output = {"status": "tracer-error", "error": "%s: %s" % (type(error).__name__, error)}
+    with open(config["result_path"], "w") as handle:
+        json.dump(output, handle)
+
+
+def __lldb_init_module(debugger, internal_dict):
+    debugger.HandleCommand("command script add -f rea_lldb_tracer.run rea_trace")

package/bridge/process/ProcessRunTokenReader.swift

@@ -0,0 +1,263 @@
+import Darwin
+import Foundation
+
+struct RunTokenObservation: Encodable {
+  let pid: Int32
+  let state: String
+  let run_id: String?
+  let reason: String?
+}
+
+struct ProcessIdentityObservation: Encodable {
+  let pid: Int32
+  let state: String
+  let identity: String?
+  let reason: String?
+}
+
+private struct ProcessMetadata {
+  let identity: String
+  let pointerSize: Int
+}
+
+struct ProcessSnapshot {
+  fileprivate let metadata: [Int32: ProcessMetadata]
+  let failure: String?
+}
+
+typealias ProcessTableSysctlCall = (
+  _ mib: inout [Int32],
+  _ output: UnsafeMutableRawPointer?,
+  _ size: inout Int
+) -> Int32
+
+private func readProcessSnapshot() -> ProcessSnapshot {
+  readProcessSnapshot { mib, output, size in
+    sysctl(&mib, u_int(mib.count), output, &size, nil, 0)
+  }
+}
+
+func readProcessSnapshot(
+  sysctlCall: ProcessTableSysctlCall
+) -> ProcessSnapshot {
+  for _ in 0..<3 {
+    var mib = [CTL_KERN, KERN_PROC, KERN_PROC_ALL, 0]
+    var size = 0
+    guard sysctlCall(&mib, nil, &size) == 0 else {
+      let failure = errno
+      return ProcessSnapshot(metadata: [:], failure: "process_table_failed_\(failure)")
+    }
+    guard size >= MemoryLayout<kinfo_proc>.size,
+          size % MemoryLayout<kinfo_proc>.stride == 0 else {
+      return ProcessSnapshot(metadata: [:], failure: "process_table_failed_\(errno)")
+    }
+    var processes = [kinfo_proc](
+      repeating: kinfo_proc(),
+      count: size / MemoryLayout<kinfo_proc>.stride
+    )
+    let result = processes.withUnsafeMutableBufferPointer { buffer in
+      let output = buffer.baseAddress.map { UnsafeMutableRawPointer($0) }
+      return sysctlCall(&mib, output, &size)
+    }
+    guard result == 0 else {
+      let failure = errno
+      if failure == ENOMEM { continue }
+      return ProcessSnapshot(metadata: [:], failure: "process_table_failed_\(failure)")
+    }
+    guard size % MemoryLayout<kinfo_proc>.stride == 0 else {
+      return ProcessSnapshot(metadata: [:], failure: "process_table_failed_\(errno)")
+    }
+    let processCount = size / MemoryLayout<kinfo_proc>.stride
+    let metadata = Dictionary(
+      processes.prefix(processCount).map { process in
+        let start = process.kp_proc.p_starttime
+        return (
+          process.kp_proc.p_pid,
+          ProcessMetadata(
+            identity: "\(start.tv_sec):\(start.tv_usec)",
+            pointerSize: process.kp_proc.p_flag & P_LP64 != 0 ? 8 : 4
+          )
+        )
+      },
+      uniquingKeysWith: { first, _ in first }
+    )
+    return ProcessSnapshot(metadata: metadata, failure: nil)
+  }
+  return ProcessSnapshot(metadata: [:], failure: "process_table_failed_\(ENOMEM)")
+}
+
+func readProcessIdentities(pids: [Int32]) -> [ProcessIdentityObservation] {
+  let snapshot = readProcessSnapshot()
+  return pids.map { pid in
+    guard let metadata = snapshot.metadata[pid] else {
+      return unavailableIdentity(
+        pid: pid,
+        reason: snapshot.failure ?? "process_not_in_snapshot"
+      )
+    }
+    return ProcessIdentityObservation(pid: pid, state: "readable", identity: metadata.identity, reason: nil)
+  }
+}
+
+private func unavailableIdentity(pid: Int32, reason: String) -> ProcessIdentityObservation {
+  ProcessIdentityObservation(pid: pid, state: "unavailable", identity: nil, reason: reason)
+}
+
+enum RunTokenReadError: Error {
+  case invalidBuffer
+  case duplicateToken
+  case invalidTokenEncoding
+  case environmentUnavailable
+  case appleVectorUnavailable
+  case ambiguousEnvironmentBoundary
+}
+
+private func readRunToken(pid: Int32, pointerSize: Int) -> RunTokenObservation {
+  var mib = [CTL_KERN, KERN_PROCARGS2, Int32(pid)]
+  var size = 0
+  guard sysctl(&mib, u_int(mib.count), nil, &size, nil, 0) == 0, size >= MemoryLayout<Int32>.size else {
+    return unavailable(pid: pid, reason: "sysctl_failed_\(errno)")
+  }
+  var bytes = [UInt8](repeating: 0, count: size)
+  let result = bytes.withUnsafeMutableBytes { buffer in
+    sysctl(&mib, u_int(mib.count), buffer.baseAddress, &size, nil, 0)
+  }
+  guard result == 0 else {
+    return unavailable(pid: pid, reason: "sysctl_failed_\(errno)")
+  }
+  bytes = Array(bytes.prefix(size))
+  do {
+    return RunTokenObservation(pid: pid, state: "readable", run_id: try token(in: bytes, pointerSize: pointerSize), reason: nil)
+  } catch RunTokenReadError.duplicateToken {
+    return unavailable(pid: pid, reason: "duplicate_run_token")
+  } catch RunTokenReadError.invalidTokenEncoding {
+    return unavailable(pid: pid, reason: "invalid_run_token_encoding")
+  } catch RunTokenReadError.environmentUnavailable {
+    return unavailable(pid: pid, reason: "environment_unavailable")
+  } catch RunTokenReadError.appleVectorUnavailable {
+    return unavailable(pid: pid, reason: "apple_vector_unavailable")
+  } catch RunTokenReadError.ambiguousEnvironmentBoundary {
+    return unavailable(pid: pid, reason: "ambiguous_environment_boundary")
+  } catch {
+    return unavailable(pid: pid, reason: "malformed_procargs")
+  }
+}
+
+func readRunTokens(pids: [Int32]) -> [RunTokenObservation] {
+  let before = readProcessSnapshot()
+  let observations = pids.map { pid -> RunTokenObservation in
+    guard let metadata = before.metadata[pid] else {
+      return unavailable(pid: pid, reason: before.failure ?? "process_not_in_snapshot")
+    }
+    return readRunToken(pid: pid, pointerSize: metadata.pointerSize)
+  }
+  let after = readProcessSnapshot()
+  return zip(pids, observations).map { pid, observation in
+    guard let initial = before.metadata[pid], let current = after.metadata[pid] else {
+      return unavailable(pid: pid, reason: after.failure ?? "process_not_in_snapshot")
+    }
+    guard initial.identity == current.identity,
+          initial.pointerSize == current.pointerSize else {
+      return unavailable(pid: pid, reason: "process_identity_changed_during_token_read")
+    }
+    return observation
+  }
+}
+
+func token(in bytes: [UInt8], pointerSize: Int = MemoryLayout<UnsafeRawPointer>.size) throws -> String? {
+  guard bytes.count >= MemoryLayout<Int32>.size else { throw RunTokenReadError.invalidBuffer }
+  let argc = bytes.withUnsafeBytes { $0.loadUnaligned(as: Int32.self).littleEndian }
+  guard argc >= 0, Int(argc) <= bytes.count else { throw RunTokenReadError.invalidBuffer }
+  var offset = MemoryLayout<Int32>.size
+  guard let executablePath = readString(in: bytes, offset: &offset) else {
+    throw RunTokenReadError.invalidBuffer
+  }
+  guard pointerSize == 4 || pointerSize == 8 else { throw RunTokenReadError.invalidBuffer }
+  let pathRecordSize = Array("executable_path=".utf8).count + executablePath.count + 1
+  let padding = (pointerSize - pathRecordSize % pointerSize) % pointerSize
+  guard offset + padding <= bytes.count,
+        bytes[offset..<(offset + padding)].allSatisfy({ $0 == 0 }) else {
+    throw RunTokenReadError.invalidBuffer
+  }
+  offset += padding
+  for _ in 0..<argc {
+    guard readString(in: bytes, offset: &offset, allowEmpty: true) != nil else {
+      throw RunTokenReadError.invalidBuffer
+    }
+  }
+  guard bytes.last == 0 else { throw RunTokenReadError.invalidBuffer }
+  if offset == bytes.count {
+    throw RunTokenReadError.environmentUnavailable
+  }
+  var trailingEntries: [ArraySlice<UInt8>] = []
+  var zeroedAppleVectorStart: Int?
+  while offset < bytes.count {
+    if bytes[offset] == 0 {
+      let start = offset
+      while offset < bytes.count, bytes[offset] == 0 {
+        offset += 1
+      }
+      if offset - start > 1 {
+        zeroedAppleVectorStart = zeroedAppleVectorStart ?? trailingEntries.count
+      }
+      continue
+    }
+    guard let entry = readString(in: bytes, offset: &offset) else {
+      throw RunTokenReadError.invalidBuffer
+    }
+    trailingEntries.append(entry)
+  }
+  // XNU appends its Apple vector after the environment. libc clears the
+  // initial `pfz=` record after startup, leaving a multi-NUL run in its place.
+  // A caller can also set `pfz`, so prefer the final intact sentinel when it
+  // remains available and otherwise stop at the cleared record.
+  let intactAppleVectorStart = trailingEntries.lastIndex(where: {
+    $0.starts(with: Array("pfz=".utf8))
+  })
+  let preferZeroedBoundary = zeroedAppleVectorStart.flatMap { zeroedStart in
+    intactAppleVectorStart.map { intactStart in intactStart < zeroedStart } ?? true
+  } ?? false
+  let appleVectorStart = preferZeroedBoundary
+    ? zeroedAppleVectorStart
+    : intactAppleVectorStart
+  guard let appleVectorStart else {
+    throw RunTokenReadError.appleVectorUnavailable
+  }
+  if trailingEntries[appleVectorStart...].contains(where: {
+    $0.starts(with: Array("REA_PROCESS_RUN_ID=".utf8))
+  }) {
+    throw RunTokenReadError.ambiguousEnvironmentBoundary
+  }
+  let environmentEntries = trailingEntries[..<appleVectorStart]
+  guard !environmentEntries.isEmpty else {
+    throw RunTokenReadError.environmentUnavailable
+  }
+  var found: String?
+  for entry in environmentEntries {
+    if entry.starts(with: Array("REA_PROCESS_RUN_ID=".utf8)) {
+      guard found == nil else { throw RunTokenReadError.duplicateToken }
+      let value = entry.dropFirst("REA_PROCESS_RUN_ID=".utf8.count)
+      guard let decoded = String(bytes: value, encoding: .utf8) else {
+        throw RunTokenReadError.invalidTokenEncoding
+      }
+      found = decoded
+    }
+  }
+  return found
+}
+
+private func readString(
+  in bytes: [UInt8],
+  offset: inout Int,
+  allowEmpty: Bool = false
+) -> ArraySlice<UInt8>? {
+  guard offset < bytes.count, let end = bytes[offset...].firstIndex(of: 0),
+        allowEmpty || end > offset else { return nil }
+  let string = bytes[offset..<end]
+  offset = end + 1
+  return string
+}
+
+private func unavailable(pid: Int32, reason: String) -> RunTokenObservation {
+  RunTokenObservation(pid: pid, state: "unavailable", run_id: nil, reason: reason)
+}

package/bridge/process/ProcessRunTokenReaderMain.swift

@@ -0,0 +1,35 @@
+import Foundation
+
+struct RunTokenResponse: Encodable {
+  let results: [RunTokenObservation]
+}
+
+struct ProcessIdentityResponse: Encodable {
+  let results: [ProcessIdentityObservation]
+}
+
+@main
+struct ProcessRunTokenReaderMain {
+  static func main() {
+    let arguments = Array(CommandLine.arguments.dropFirst())
+    let identityMode = arguments.first == "--identities"
+    let pidArguments = identityMode ? Array(arguments.dropFirst()) : arguments
+    let pids = pidArguments.compactMap(Int32.init)
+    guard pids.count == pidArguments.count else {
+      fputs("invalid process ID input\n", stderr)
+      exit(64)
+    }
+    do {
+      let output: Data
+      if identityMode {
+        output = try JSONEncoder().encode(ProcessIdentityResponse(results: readProcessIdentities(pids: pids)))
+      } else {
+        output = try JSONEncoder().encode(RunTokenResponse(results: readRunTokens(pids: pids)))
+      }
+      FileHandle.standardOutput.write(output)
+    } catch {
+      fputs("could not encode process token observations\n", stderr)
+      exit(70)
+    }
+  }
+}