scientific-method-engine 6.3.0__tar.gz → 7.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/PKG-INFO +1 -1
  2. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/pyproject.toml +1 -1
  3. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/cli.py +2 -1
  4. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/x86/reports.py +209 -43
  5. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/x86/trace.py +20 -4
  6. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/tests/test_pe.py +114 -0
  7. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/tests/test_x86.py +93 -13
  8. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/.gitignore +0 -0
  9. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/LICENSE +0 -0
  10. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/README.md +0 -0
  11. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/__init__.py +0 -0
  12. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/__main__.py +0 -0
  13. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  14. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  15. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  16. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  17. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  18. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  19. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  20. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  21. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  22. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  23. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  24. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  25. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  26. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  27. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  28. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  29. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  30. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  31. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  32. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  33. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  34. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  35. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  36. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  37. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  38. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  39. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  40. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  41. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  42. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  43. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
  44. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  45. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
  46. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  47. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  48. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/x86/image.py +0 -0
  49. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/x86/loops.py +0 -0
  50. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/x86/machine.py +0 -0
  51. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
  52. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/x86/pcode.py +0 -0
  53. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
  54. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/x86/pe.py +0 -0
  55. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/x86/relational.py +0 -0
  56. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  57. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/src/scientific_method_engine/x86/values.py +0 -0
  58. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/tests/oracle.py +0 -0
  59. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/tests/test_continuation_budget.py +0 -0
  60. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/tests/test_dispatch.py +0 -0
  61. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/tests/test_effect_order.py +0 -0
  62. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/tests/test_entry_frame.py +0 -0
  63. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/tests/test_loops.py +0 -0
  64. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/tests/test_memory_scopes.py +0 -0
  65. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/tests/test_nested_frame_request.py +0 -0
  66. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/tests/test_oracle.py +0 -0
  67. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/tests/test_relational_controls.py +0 -0
  68. {scientific_method_engine-6.3.0 → scientific_method_engine-7.1.0}/tests/test_table_continuations.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 6.3.0
3
+ Version: 7.1.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "6.3.0"
8
+ version = "7.1.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -17,7 +17,8 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
17
17
  "effects|returns|memory|incoming|call-order|guards|allocation|dispatch> <config.json|->\n"
18
18
  "effects includes ordered path writes/calls and local restoration witnesses; transactionality remains unestablished.\n"
19
19
  "callees compares its edges with an ExportCallEdges.java export given as ghidraCallEdges;\n"
20
- "agreed is false where Ghidra ends the function at a call, conditional jump or interrupt the engine reads past (ghidraEndsFunction).\n"
20
+ "agreed is false where Ghidra ends the function at a call, conditional jump or interrupt the engine reads past (ghidraEndsFunction),\n"
21
+ "or continues past a jmp, ljmp, return or hlt the engine stops at (ghidraContinues).\n"
21
22
  "trace, arguments, effects, returns, guards, memory and allocation check relationalControls:\n"
22
23
  "a violated control fails the report; an undecided one is reported and never counts as held.\n"
23
24
  "entryFrame {from} starts an entry inside the function at from, in the frame a trace from there observed.\n"
@@ -204,6 +204,107 @@ CFG_OPERAND = "entry-CFG operand past a stop; values and callee effects unresolv
204
204
  PORT_OPERAND = "operand past a PE32 port access; values and continuation unresolved"
205
205
 
206
206
 
207
+ # dependsOn reasons in ``uses`` for the calls and port accesses a conditionalAccesses row is reached past.
208
+ STEPPED_CALL = "call past a stop; assumed to return"
209
+ STEPPED_PORT = "port access past a stop; assumed to continue"
210
+ OPEN_CALL = "call open at a stop inside its callee; continued at its return site, assumed to return"
211
+
212
+
213
+ def _function_exit(image, start, limit, follow_flat_ports, cache):
214
+ """Walk one function's CFG from ``start``, stepping over calls, and say whether it reaches a return.
215
+
216
+ Returns the decoded sites on a route from ``start`` to a return instruction, whether one was
217
+ reached, and whether the walk stopped at ``limit`` first. A PE32 port access ends its branch
218
+ unless ``follow_flat_ports``. In the PE32 model an IRET stops ``trace``, so it is no return here.
219
+ ``cache`` keeps each walk's result, as several open calls and stops share the same walks.
220
+ """
221
+ key = (start, follow_flat_ports)
222
+ if key in cache:
223
+ return cache[key]
224
+ pending, seen, successors, exits = [start], {}, {}, []
225
+ while pending:
226
+ at = pending.pop()
227
+ if at in seen:
228
+ continue
229
+ if len(seen) >= limit:
230
+ cache[key] = {}, False, True
231
+ return cache[key]
232
+ ins = image.decode(at)
233
+ if ins is None:
234
+ continue
235
+ seen[at] = ins
236
+ m = base_mnemonic(ins)
237
+ if unsupported_transfer(image, ins):
238
+ continue
239
+ following = successors[at] = []
240
+ if at in image.indirect_jumps:
241
+ following.extend(row["target"] for row in image.indirect_jumps[at]["rows"])
242
+ elif m in RETURNS:
243
+ if not (image.flat and m in ("iret", "iretd")):
244
+ exits.append(at)
245
+ else:
246
+ if m == "ljmp" or m.startswith(("j", "loop")):
247
+ target, _ = call_target(image, at, ins)
248
+ if target is not None:
249
+ following.append(target)
250
+ if not (m in ("jmp", "ljmp") or m in INTERRUPTS or m == "hlt"
251
+ or (m in PORTS and image.flat and not follow_flat_ports)):
252
+ following.append(at + ins.size)
253
+ pending.extend(following)
254
+ # Only the sites a return is reachable from lie on the way to it; a branch that never returns is left out.
255
+ callers = {}
256
+ for at, following in successors.items():
257
+ for target in following:
258
+ callers.setdefault(target, []).append(at)
259
+ on_route, pending = set(exits), list(exits)
260
+ while pending:
261
+ for at in callers.get(pending.pop(), ()):
262
+ if at not in on_route:
263
+ on_route.add(at)
264
+ pending.append(at)
265
+ cache[key] = {at: seen[at] for at in on_route}, bool(exits), False
266
+ return cache[key]
267
+
268
+
269
+ def _caller_continuations(image, stop, reason, stack, limit, cache):
270
+ """Return sites at which ``uses`` continues its inventory past a stop inside a called function.
271
+
272
+ ``stack`` holds the traced calls still open at ``stop`` as (call site, return site) pairs,
273
+ outermost first. A return site is continued only when the CFG from the stop, or from the
274
+ return site inside it, reaches a return of the called function. Each continuation depends on
275
+ the stop, on every open call from the stop out to that return site, and on every call or PE32
276
+ port access stepped over on the way to those returns. Returns (return site, dependsOn,
277
+ reached without crossing a PE32 port access) rows and the gaps of walks that reached ``limit``.
278
+ """
279
+ rows = []
280
+ ins = image.decode(stop)
281
+ # A stop at a return instruction is the return itself failing, so no caller continuation is assumed.
282
+ if ins is None or base_mnemonic(ins) in RETURNS:
283
+ return rows, []
284
+ depends = [{"site": stop, "reason": reason}]
285
+ start, port_free = stop, not (image.flat and base_mnemonic(ins) in PORTS)
286
+ for call_site, return_site in reversed(stack):
287
+ route, exits, truncated = _function_exit(image, start, limit, True, cache)
288
+ if truncated:
289
+ return rows, [{"site": start, "reason": "instruction limit"}]
290
+ if not exits:
291
+ break
292
+ if port_free:
293
+ # This walk decodes a subset of the one above, so it cannot reach the limit.
294
+ port_free = _function_exit(image, start, limit, False, cache)[1]
295
+ for at, ins in sorted(route.items()):
296
+ if at == stop:
297
+ continue
298
+ if ins.mnemonic in ("call", "lcall"):
299
+ depends.append({"site": at, "reason": STEPPED_CALL})
300
+ elif image.flat and base_mnemonic(ins) in PORTS:
301
+ depends.append({"site": at, "reason": STEPPED_PORT})
302
+ depends.append({"site": call_site, "reason": OPEN_CALL})
303
+ start = return_site
304
+ rows.append((start, list(depends), port_free))
305
+ return rows, []
306
+
307
+
207
308
  def uses(image, config):
208
309
  query = config.get("query", {})
209
310
  offset = integer(query.get("offset"), 0, image.mask, "query offset")
@@ -233,6 +334,8 @@ def uses(image, config):
233
334
  entry_limit = integer(config.get("entryLimit", 64), 1, 256, "entryLimit")
234
335
  # CFG points where value propagation stopped (or never started), with why; operands after them are inventoried below.
235
336
  stops = {}
337
+ # The traced calls still open at each stop, so the inventory can continue at their return sites.
338
+ open_calls = {}
236
339
  established = entries(image)
237
340
  for index, at in enumerate(established):
238
341
  if remaining <= 0 or index >= entry_limit:
@@ -241,17 +344,18 @@ def uses(image, config):
241
344
  stops.setdefault(root, "entry not traced: entry or total instruction budget exhausted")
242
345
  break
243
346
  report = trace(image, {**config, "entry": at, "totalSteps": remaining, "stringIterations": string_remaining},
244
- continue_declared_jumps=False, track_loops=False)
347
+ continue_declared_jumps=False, track_loops=False, call_stacks=True)
245
348
  remaining -= report["stepsUsed"]
246
349
  string_remaining -= report["stringIterationsUsed"]
247
350
  if not report["completeWithinModel"]:
248
351
  gaps.append({"entry": at, "reason": "incomplete path effects", "stops": list({p["stop"] for p in report["paths"] if p["stop"]})})
249
- for p in report["paths"]:
250
- if p["stop"] and p["stopSite"] is not None:
251
- stops.setdefault(p["stopSite"], p["stop"])
252
- for g in report["gaps"]:
253
- if "site" in g:
254
- stops.setdefault(g["site"], g["reason"])
352
+ stopped = [(p["stopSite"], p["stop"], p["callStack"]) for p in report["paths"]
353
+ if p["stop"] and p["stopSite"] is not None]
354
+ stopped += [(g["site"], g["reason"], g.get("callStack")) for g in report["gaps"] if "site" in g]
355
+ for site, reason, stack in stopped:
356
+ stops.setdefault(site, reason)
357
+ if stack:
358
+ open_calls.setdefault(site, set()).add(tuple((f["callSite"], f["continuation"]) for f in stack))
255
359
  for path in report["paths"]:
256
360
  for e in path["events"]:
257
361
  if e["kind"] not in ("read", "write") or mode not in ("both", e["kind"]):
@@ -289,29 +393,51 @@ def uses(image, config):
289
393
  instruction_limit = config.get("instructionLimit", 10000)
290
394
  # This inventory assumes execution continues past each stop, so it also follows PE32 port accesses
291
395
  # and names each one below.
292
- after_stop, stop_gaps, _, _, _ = (walk(image, list(stops), instruction_limit, follow_flat_ports=True)
293
- if stops else ({}, [], None, None, None))
396
+ # A stop inside a called function would end the inventory at that function's return. The code
397
+ # after each call still open at the stop is inventoried too, from the call's return site, and
398
+ # depends on the stop and on every call between them returning.
399
+ returning, exit_walks = [], {}
400
+ for root, stacks in sorted(open_calls.items()):
401
+ for stack in sorted(stacks):
402
+ rows, frame_gaps = _caller_continuations(image, root, stops[root], stack, instruction_limit, exit_walks)
403
+ returning.extend(rows)
404
+ gaps.extend(g for g in frame_gaps if g not in gaps)
405
+ seeds = list(stops) + [start for start, _, _ in returning]
406
+ after_stop, stop_gaps, _, _, _ = (walk(image, seeds, instruction_limit, follow_flat_ports=True)
407
+ if seeds else ({}, [], None, None, None))
294
408
  gaps.extend(g for g in stop_gaps if g["reason"] == "instruction limit")
295
409
  # In PE32 a site the stops reach only by continuing past a port access is named as such, even when
296
410
  # it also depends on an unread call. The walk that ends at port accesses must finish within the
297
- # limit for that claim; otherwise every row keeps the shared value.
411
+ # limit for that claim; otherwise every row keeps the shared value. A return site counts as
412
+ # reached without a port access only when its callees return without crossing one.
298
413
  port_only = set()
299
- if image.flat and any(base_mnemonic(ins) in PORTS for ins in after_stop.values()):
300
- before_ports, port_gaps, _, _, _ = walk(image, list(stops), instruction_limit)
414
+ if image.flat and (any(base_mnemonic(ins) in PORTS for ins in after_stop.values())
415
+ or any(not port_free for _, _, port_free in returning)):
416
+ port_free_seeds = list(stops) + [start for start, _, port_free in returning if port_free]
417
+ before_ports, port_gaps, _, _, _ = walk(image, port_free_seeds, instruction_limit)
301
418
  if not any(g["reason"] == "instruction limit" for g in port_gaps):
302
419
  port_only = set(after_stop) - set(before_ports)
303
420
  # A call past a stop was never traced either, so code after it also depends on it returning.
304
- starts = [(root, root, reason) for root, reason in stops.items()]
305
- starts += [(at, at + ins.size, "call past a stop; assumed to return")
421
+ starts = [(root, [{"site": root, "reason": reason}]) for root, reason in stops.items()]
422
+ starts += [(at + ins.size, [{"site": at, "reason": STEPPED_CALL}])
306
423
  for at, ins in after_stop.items() if ins.mnemonic in ("call", "lcall") and at not in stops]
307
424
  if image.flat:
308
- starts += [(at, at + ins.size, "port access past a stop; assumed to continue")
425
+ starts += [(at + ins.size, [{"site": at, "reason": STEPPED_PORT}])
309
426
  for at, ins in after_stop.items() if base_mnemonic(ins) in PORTS and at not in stops]
427
+ starts += [(start, row_depends) for start, row_depends, _ in returning]
428
+ # Several rows can share a start (one return site of many open calls), so each start is walked once.
429
+ by_start = {}
430
+ for start, row_depends in starts:
431
+ named = by_start.setdefault(start, [])
432
+ named.extend(d for d in row_depends if d not in named)
310
433
  depends = {}
311
- for site, start, reason in sorted(starts):
434
+ for start, row_depends in by_start.items():
312
435
  reached, _, _, _, _ = walk(image, [start], instruction_limit, follow_flat_ports=True)
313
436
  for at in reached:
314
- depends.setdefault(at, []).append({"site": site, "reason": reason})
437
+ named = depends.setdefault(at, [])
438
+ named.extend(d for d in row_depends if d not in named)
439
+ for named in depends.values():
440
+ named.sort(key=lambda d: (d["site"], d["reason"]))
315
441
  conditional = []
316
442
  for at, ins in sorted(after_stop.items()):
317
443
  if ins.mnemonic == "lea":
@@ -1041,7 +1167,7 @@ def callees(image, config):
1041
1167
  and not s["omittedRoutes"]
1042
1168
  and not any(d.get("reason") in capped for at in s["entries"] for d in nodes[at]["dependencies"])
1043
1169
  and not any(d.get("reason") in capped for i in s["dependencyEdges"] for d in edges[i]["dependencies"]))
1044
- cross_check = _ghidra_cross_check(export, nodes, outgoing, omitted) if export is not None else None
1170
+ cross_check = _ghidra_cross_check(image, export, nodes, outgoing, omitted) if export is not None else None
1045
1171
  known = {"sharedSites": {e["site"] for e in edges if shared_control(e)},
1046
1172
  "recursiveSites": {e["site"] for e in edges if e["classification"] == "recursivePath"},
1047
1173
  "writeSites": {o["site"] for n in nodes.values() for o in n["memoryObservations"] if o["boundaryUsable"] and "write" in o["access"]},
@@ -1121,22 +1247,60 @@ def _ghidra_key(g):
1121
1247
  return g["site"], g["target"]
1122
1248
 
1123
1249
 
1250
+ # The flow types Ghidra's RefType builds with a fall-through (FlowType.hasFallthrough). Every other flow type ends the
1251
+ # function at its instruction, CONDITIONAL_CALL_TERMINATOR included.
1252
+ GHIDRA_FALL_THROUGH_FLOWS = frozenset((
1253
+ "FALL_THROUGH", "CONDITIONAL_JUMP", "UNCONDITIONAL_CALL", "CONDITIONAL_CALL", "CONDITIONAL_TERMINATOR",
1254
+ "COMPUTED_CALL", "CONDITIONAL_COMPUTED_CALL", "CONDITIONAL_COMPUTED_JUMP", "CALL_OVERRIDE_UNCONDITIONAL",
1255
+ "CALLOTHER_OVERRIDE_CALL"))
1256
+
1257
+
1124
1258
  def _ghidra_falls_through(g):
1125
1259
  """Whether Ghidra continues to the next instruction at an exported edge's site, and what that was read from.
1126
1260
 
1127
1261
  The export's fallsThrough also reflects a user's fall-through override. Copies of the script that leave it out
1128
- are read by the flow type's name, which misses such an override.
1262
+ are read by the flow type's name, which misses such an override: a flow type Ghidra gives a fall-through
1263
+ (GHIDRA_FALL_THROUGH_FLOWS) continues, and every other one ends the function.
1129
1264
  """
1130
1265
  exported = g["fallsThrough"] is not None
1131
- return {"ghidraFallsThrough": g["fallsThrough"] if exported else "TERMINATOR" not in g["flow"],
1266
+ named = g["flow"] in GHIDRA_FALL_THROUGH_FLOWS
1267
+ return {"ghidraFallsThrough": g["fallsThrough"] if exported else named,
1132
1268
  "ghidraFallsThroughBasis": "fallsThrough" if exported else "flowName"}
1133
1269
 
1134
1270
 
1135
- def _ghidra_cross_check(export, nodes, outgoing, omitted):
1271
+ def _engine_reads_on(image, ins):
1272
+ """Whether the engine's body reading continues to the instruction after ins.
1273
+
1274
+ It stops at a jmp, ljmp, return or hlt, and reads on past every other instruction, including a call, a conditional
1275
+ jump and an interrupt. At a transfer outside the frame model it stops with a gap, so the result is None: the engine
1276
+ decided nothing there. It is None too when the engine did not read ins.
1277
+ """
1278
+ if ins is None or unsupported_transfer(image, ins):
1279
+ return None
1280
+ m = base_mnemonic(ins)
1281
+ return not (m in RETURNS or m in ("hlt", "jmp", "ljmp"))
1282
+
1283
+
1284
+ def _ghidra_cross_check(image, export, nodes, outgoing, omitted):
1136
1285
  """Compare the engine's edges with Ghidra's for each caller both read; a Ghidra-only edge stays unchecked."""
1137
1286
  callers = export["callers"]
1138
1287
  compared = sorted(nodes.keys() & callers.keys())
1139
1288
  rows = []
1289
+ # Rows where Ghidra ends the function at an instruction the engine reads past, and rows where Ghidra continues past
1290
+ # an instruction the engine stops at. Either way the two analyses disagree on the function's extent.
1291
+ ends, continues = [], []
1292
+
1293
+ def compare_extent(row, g, ins):
1294
+ reads_on = _engine_reads_on(image, ins)
1295
+ if reads_on is None:
1296
+ return row
1297
+ row |= _ghidra_falls_through(g)
1298
+ if reads_on and row["ghidraFallsThrough"] is False:
1299
+ ends.append(row)
1300
+ elif not reads_on and row["ghidraFallsThrough"] is True:
1301
+ continues.append(row)
1302
+ return row
1303
+
1140
1304
  for caller in compared:
1141
1305
  ours = outgoing.get(caller, [])
1142
1306
  theirs = callers[caller]
@@ -1148,12 +1312,9 @@ def _ghidra_cross_check(export, nodes, outgoing, omitted):
1148
1312
  g = matches.get((e["site"], e["target"]))
1149
1313
  row = {"caller": caller, "site": e["site"], "target": e["target"], "engineEdge": e["id"],
1150
1314
  "result": "engineOnly" if g is None else "agreement", "ghidraFlow": g["flow"] if g else None}
1151
- # The engine reads on past every call and every conditional tail transfer, and stops at a jmp or ljmp.
1152
1315
  # Ghidra ends the function at a call to a callee it treats as non-returning (CALL_TERMINATOR) or at an
1153
- # instruction whose fall-through a user cleared.
1154
- if g is not None and (e["kind"] != "tail transfer" or base_mnemonic(instructions[e["site"]]) not in ("jmp", "ljmp")):
1155
- row |= _ghidra_falls_through(g)
1156
- rows.append(row)
1316
+ # instruction whose fall-through a user cleared, and continues past a jmp a user gave a fall-through.
1317
+ rows.append(row if g is None else compare_extent(row, g, instructions[e["site"]]))
1157
1318
  for g in theirs:
1158
1319
  if g["site"] is not None and _ghidra_key(g) in read:
1159
1320
  continue
@@ -1162,35 +1323,40 @@ def _ghidra_cross_check(export, nodes, outgoing, omitted):
1162
1323
  # SLEIGH lifts INT, INT1, INT3 and INTO to a computed call with no target, while the engine assumes the
1163
1324
  # interrupt returns to the next instruction and records no edge. At INT1 and INT3 Ghidra's flow is a
1164
1325
  # terminator that ends the function there.
1165
- rows.append({"caller": caller, "site": g["site"], "target": None, "siteAddress": g["siteAddress"],
1166
- "targetAddress": None, "ghidraFlow": g["flow"], "result": "interrupt", "engineEdge": None,
1167
- **_ghidra_falls_through(g)})
1326
+ rows.append(compare_extent({"caller": caller, "site": g["site"], "target": None, "siteAddress": g["siteAddress"],
1327
+ "targetAddress": None, "ghidraFlow": g["flow"], "result": "interrupt",
1328
+ "engineEdge": None}, g, ins))
1168
1329
  continue
1169
- # Ghidra's edge is evidence the engine did not check; it never becomes an engine edge.
1170
- rows.append({"caller": caller, "site": g["site"], "target": g["target"], "siteAddress": g["siteAddress"],
1171
- "targetAddress": g["targetAddress"], "ghidraFlow": g["flow"], "result": "ghidraOnly", "checked": False,
1172
- "engineEdge": next((e["id"] for e in ours if g["site"] is not None and e["site"] == g["site"]), None)})
1330
+ # Ghidra's edge is evidence the engine did not check; it never becomes an engine edge. Its fall-through is
1331
+ # still compared where the engine read the instruction at its site.
1332
+ engine_edge = next((e["id"] for e in ours if g["site"] is not None and e["site"] == g["site"]), None)
1333
+ rows.append(compare_extent({"caller": caller, "site": g["site"], "target": g["target"], "siteAddress": g["siteAddress"],
1334
+ "targetAddress": g["targetAddress"], "ghidraFlow": g["flow"], "result": "ghidraOnly",
1335
+ "checked": False, "engineEdge": engine_edge}, g, ins))
1173
1336
  counts = {kind: sum(r["result"] == kind for r in rows) for kind in ("agreement", "engineOnly", "ghidraOnly", "interrupt")}
1174
- counts["ghidraEndsFunction"] = sum(r.get("ghidraFallsThrough") is False for r in rows)
1337
+ counts["ghidraEndsFunction"] = len(ends)
1338
+ counts["ghidraContinues"] = len(continues)
1175
1339
  not_compared = {"engineCallers": sorted(nodes.keys() - callers.keys()), "ghidraCallers": sorted(callers.keys() - nodes.keys()),
1176
1340
  "unmappedGhidraFunctions": export["unmappedFunctions"], "missingGhidraEntries": export["missingEntries"],
1177
1341
  "unreadGhidraFunctions": export["unreadFunctions"],
1178
1342
  "omittedEngineRoutes": [o["id"] for o in omitted if o["entry"] in callers]}
1179
- # A site agrees only when every edge either analysis read there agrees and Ghidra continues past it as the engine does.
1180
- disputed = {r["site"] for r in rows if r["result"] != "agreement" or r.get("ghidraFallsThrough") is False}
1343
+ # A site agrees only when every edge either analysis read there agrees and Ghidra continues past it exactly when the
1344
+ # engine does.
1345
+ disputed = {r["site"] for r in rows if r["result"] != "agreement"} | {r["site"] for r in ends + continues}
1181
1346
  return {"comparedCallers": compared, "edges": rows, "counts": counts, "notCompared": not_compared,
1182
1347
  "agreed": (not counts["engineOnly"] and not counts["ghidraOnly"] and not any(not_compared.values())
1183
- and not counts["ghidraEndsFunction"]),
1348
+ and not counts["ghidraEndsFunction"] and not counts["ghidraContinues"]),
1184
1349
  "agreementSites": {r["site"] for r in rows if r["result"] == "agreement"} - disputed,
1185
1350
  "interpretation": "Edges of each caller that both the engine and the Ghidra export read, matched by site and target "
1186
1351
  "file offset; an unresolved call matches an unresolved call at its site, and a Ghidra target without a file offset "
1187
1352
  "matches no engine edge. An interrupt row is Ghidra's targetless call at an instruction the engine read as an "
1188
- "interrupt and assumed to return. The engine reads on past every call, conditional jump and interrupt, so an "
1189
- "interrupt row or an agreement at a call or conditional tail transfer counts against agreed when Ghidra "
1190
- "ends the function there (ghidraFallsThrough false, counted in ghidraEndsFunction), read from the export's "
1191
- "fallsThrough or, in an export without it, from the flow name (ghidraFallsThroughBasis). "
1192
- "A ghidraOnly edge is Ghidra's claim: the engine did not check it and never adds it to its graph. Agreement "
1193
- "means both analyses read the edge, not that it executes."}
1353
+ "interrupt and assumed to return. A row at an instruction the engine read carries whether Ghidra continues "
1354
+ "to the next instruction there (ghidraFallsThrough), read from the export's fallsThrough or, in an export "
1355
+ "without it, from the flow name (ghidraFallsThroughBasis). The engine reads on past every call, conditional jump "
1356
+ "and interrupt and stops at a jmp, ljmp, return or hlt. A row where Ghidra ends the function at an instruction "
1357
+ "the engine reads past (ghidraEndsFunction) or continues past one the engine stops at (ghidraContinues) "
1358
+ "counts against agreed. A ghidraOnly edge is Ghidra's claim: the engine did not check it and never adds it "
1359
+ "to its graph. Agreement means both analyses read the edge, not that it executes."}
1194
1360
 
1195
1361
 
1196
1362
  # These branches test CX/ECX (LOOPE/LOOPNE also ZF), so an adjacent CMP/TEST never describes their predicate.
@@ -371,7 +371,7 @@ def entry_frame(image, config, entry):
371
371
  "traced paths under the query's own inputs; bp null leaves BP unknown"}
372
372
 
373
373
 
374
- def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=None):
374
+ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=None, call_stacks=False):
375
375
  """Trace bounded paths, preserving declared-table continuations as separate conditional evidence.
376
376
 
377
377
  Ordinary paths run first. A path stopped at a declared indirect jump is then
@@ -382,6 +382,10 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
382
382
  Callers that never report the paths pass track_loops=False, and their paths
383
383
  carry no ``loops`` record. entry_frame passes arrive, the query entry:
384
384
  each path stops at its first arrival at the site with an ``arrival`` record.
385
+ Callers that continue past a stop at the return sites of its callers pass
386
+ call_stacks=True, and each stopped path carries ``callStack``: the traced
387
+ calls still open at the stop, outermost first, with each one's return site.
388
+ A path limit gap inside a called function carries the same ``callStack``.
385
389
  """
386
390
  entry = integer(config.get("entry"), 0, len(image.data) - 1, "entry")
387
391
  if not any(entry in r["entries"] for r in image.regions):
@@ -461,6 +465,16 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
461
465
  charge(count.number)
462
466
  string_effect(s, ins, count, remaining)
463
467
 
468
+ def open_calls(s):
469
+ return [{"callSite": f["callSite"], "continuation": f["continuation"]} for f in s.frames[1:]]
470
+
471
+ def path_limit(site, s, reason):
472
+ # A path dropped at site shares the open calls of s; a caller continuing past it needs them.
473
+ gap = {"site": site, "reason": reason}
474
+ if call_stacks and len(s.frames) > 1:
475
+ gap["callStack"] = open_calls(s)
476
+ global_gaps.append(gap)
477
+
464
478
  def finish(s, reason=None, returned=False):
465
479
  if continuing and reason and reason.startswith(STRING_BUDGET_STOP):
466
480
  # machine.py names no budget; a continuation's string iterations come from continuationBudget.
@@ -473,6 +487,8 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
473
487
  "bp": s.frame_offset(s.reg(s.bp))}
474
488
  if s.loops is not None:
475
489
  path["loops"] = s.loops.report()
490
+ if call_stacks and not returned:
491
+ path["callStack"] = open_calls(s)
476
492
  assumptions = getattr(s, "declared_jump_assumptions", [])
477
493
  if assumptions:
478
494
  path["declaredJumpAssumptions"] = assumptions
@@ -637,7 +653,7 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
637
653
  for choice in (0, 1):
638
654
  if choice == 0:
639
655
  if created >= max_paths:
640
- global_gaps.append({"site": at, "reason": "path limit at unknown direction flag"})
656
+ path_limit(at, state, "path limit at unknown direction flag")
641
657
  continue
642
658
  child = deepcopy(state); created += 1
643
659
  else:
@@ -697,7 +713,7 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
697
713
  raise StopPath("modeled far return segment changed")
698
714
  for case in model["cases"]:
699
715
  if created >= max_paths:
700
- global_gaps.append({"site": at, "reason": "path limit at modeled call"})
716
+ path_limit(at, state, "path limit at modeled call")
701
717
  break
702
718
  child = deepcopy(state)
703
719
  created += 1
@@ -847,7 +863,7 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
847
863
  state = branches.pop()
848
864
  for child in branches:
849
865
  if created >= max_paths:
850
- global_gaps.append({"site": at, "reason": "path limit"})
866
+ path_limit(at, state, "path limit")
851
867
  else:
852
868
  pending.append(child)
853
869
  created += 1
@@ -194,6 +194,120 @@ class PEReporterTests(unittest.TestCase):
194
194
  r = report('ff d0 d9 1d 00 20 40 00 c3', 'uses', query={'offset': DATA_VA, 'width': 4, 'access': 'read'})
195
195
  self.assertEqual(r['conditionalAccesses'], [])
196
196
 
197
+ def test_uses_continues_at_the_return_site_of_a_call_open_at_a_stop(self):
198
+ cfg = 'entry-CFG operand past a stop; values and callee effects unresolved'
199
+ port = 'operand past a PE32 port access; values and continuation unresolved'
200
+ port_reason = 'port access in the flat model depends on I/O privilege, which is not modeled'
201
+ call_reason = 'unresolved call: computed transfer remains unresolved'
202
+ open_call = 'call open at a stop inside its callee; continued at its return site, assumed to return'
203
+ # call f; mov [DATA_VA], eax; ret; f: <stop>; ret
204
+ for name, code, stop, reason, classification in (
205
+ ('port access', 'e8 06 00 00 00 a3 00 20 40 00 c3 66 ba c8 03 ee c3', 15, port_reason, port),
206
+ ('unread call', 'e8 06 00 00 00 a3 00 20 40 00 c3 ff d0 c3', 11, call_reason, cfg)):
207
+ with self.subTest(name):
208
+ r = report(code, 'uses', query={'offset': DATA_VA, 'width': 4}, controls=[CODE_RAW + 5])
209
+ self.assertEqual(r['matches'], [])
210
+ row, = r['conditionalAccesses']
211
+ self.assertEqual((row['site'] - CODE_RAW, row['classification']), (5, classification))
212
+ self.assertEqual([(d['site'] - CODE_RAW, d['reason']) for d in row['dependsOn']],
213
+ [(0, open_call), (stop, reason)])
214
+ self.assertFalse(r['negativeUsable'])
215
+
216
+ def test_uses_continues_at_every_return_site_of_a_nested_stop(self):
217
+ # call f; mov [DATA_VA], eax; ret; f: call g; mov [DATA_VA+4], eax; ret; g: push eax; call eax; pop eax; ret
218
+ c = Code().branch('e8', 'f').emit('a3 00 20 40 00 c3')
219
+ c.label('f').branch('e8', 'g').emit('a3 04 20 40 00 c3')
220
+ c.label('g').emit('50 ff d0 58 c3')
221
+ f, g = c.labels['f'], c.labels['g']
222
+ stop = g + 1
223
+ open_call = 'call open at a stop inside its callee; continued at its return site, assumed to return'
224
+ r = report(c, 'uses', query={'offset': DATA_VA, 'width': 8}, controls=[CODE_RAW + 5, CODE_RAW + f + 5])
225
+ rows = [(e['site'] - CODE_RAW, e['classification'], [(d['site'] - CODE_RAW, d['reason']) for d in e['dependsOn']])
226
+ for e in r['conditionalAccesses']]
227
+ unread = (stop, 'unresolved call: computed transfer remains unresolved')
228
+ cfg = 'entry-CFG operand past a stop; values and callee effects unresolved'
229
+ # Each caller's store depends on the stop and on every open call from the stop out to it.
230
+ self.assertEqual(rows, [(5, cfg, [(0, open_call), (f, open_call), unread]),
231
+ (f + 5, cfg, [(f, open_call), unread])])
232
+ self.assertFalse(r['negativeUsable'])
233
+
234
+ def test_uses_names_a_call_stepped_over_on_the_way_to_a_return_site(self):
235
+ # call f; mov [DATA_VA], eax; ret; f: call eax; call h; ret; h: ret
236
+ c = Code().branch('e8', 'f').emit('a3 00 20 40 00 c3')
237
+ c.label('f').emit('ff d0').branch('e8', 'h').emit('c3').label('h').emit('c3')
238
+ f = c.labels['f']
239
+ r = report(c, 'uses', query={'offset': DATA_VA, 'width': 4}, controls=[CODE_RAW + 5])
240
+ row, = r['conditionalAccesses']
241
+ self.assertEqual([(d['site'] - CODE_RAW, d['reason']) for d in row['dependsOn']],
242
+ [(0, 'call open at a stop inside its callee; continued at its return site, assumed to return'),
243
+ (f, 'unresolved call: computed transfer remains unresolved'),
244
+ (f + 2, 'call past a stop; assumed to return')])
245
+
246
+ def test_uses_leaves_out_a_call_on_a_branch_that_never_returns(self):
247
+ # call f; mov [DATA_VA], eax; ret; f: call eax; test eax, eax; jz L; ret; L: call g; jmp $; g: ret
248
+ c = Code().branch('e8', 'f').emit('a3 00 20 40 00 c3')
249
+ c.label('f').emit('ff d0 85 c0').branch('74', 'L').emit('c3')
250
+ c.label('L').branch('e8', 'g').emit('eb fe').label('g').emit('c3')
251
+ f = c.labels['f']
252
+ r = report(c, 'uses', query={'offset': DATA_VA, 'width': 4}, controls=[CODE_RAW + 5])
253
+ row, = r['conditionalAccesses']
254
+ self.assertEqual([(d['site'] - CODE_RAW, d['reason']) for d in row['dependsOn']],
255
+ [(0, 'call open at a stop inside its callee; continued at its return site, assumed to return'),
256
+ (f, 'unresolved call: computed transfer remains unresolved')])
257
+
258
+ def test_uses_continues_at_the_return_site_of_a_path_limit_inside_a_callee(self):
259
+ # call f; mov [ebx], eax; ret; f: test eax, eax; jz L; mov ebx, DATA_VA; ret; L: mov ebx, CODE_VA; ret
260
+ # One path: the branch that points ebx at the query is dropped at the path limit.
261
+ c = Code().branch('e8', 'f').emit('89 03 c3')
262
+ c.label('f').emit('85 c0').branch('74', 'L').emit('bb 00 20 40 00 c3').label('L').emit('bb 00 10 40 00 c3')
263
+ branch = c.labels['f'] + 2
264
+ r = report(c, 'uses', query={'offset': DATA_VA, 'width': 4}, maxPaths=1)
265
+ self.assertEqual([(e['site'] - CODE_RAW, e['address'], [(d['site'] - CODE_RAW, d['reason']) for d in e['dependsOn']])
266
+ for e in r['conditionalAccesses']],
267
+ [(5, 'possible alias',
268
+ [(0, 'call open at a stop inside its callee; continued at its return site, assumed to return'),
269
+ (branch, 'path limit')])])
270
+ self.assertFalse(r['negativeUsable'])
271
+ # The trace command's gaps carry no call stack.
272
+ self.assertNotIn('callStack', report(c, maxPaths=1)['gaps'][0])
273
+
274
+ def test_uses_does_not_continue_past_a_pe32_iret(self):
275
+ # call f; mov [DATA_VA], eax; ret; f: call eax; iretd. IRET stops the PE32 trace, so it returns to no caller.
276
+ r = report('e8 06 00 00 00 a3 00 20 40 00 c3 ff d0 cf', 'uses', query={'offset': DATA_VA, 'width': 4})
277
+ self.assertEqual(r['conditionalAccesses'], [])
278
+ self.assertFalse(r['negativeUsable'])
279
+
280
+ def test_uses_keeps_a_stop_in_the_entry_function_without_a_caller_continuation(self):
281
+ # call eax; mov [DATA_VA], eax; ret: no call is open at the stop, so dependsOn names the stop alone.
282
+ r = report('ff d0 a3 00 20 40 00 c3', 'uses', query={'offset': DATA_VA, 'width': 4}, controls=[CODE_RAW + 2])
283
+ row, = r['conditionalAccesses']
284
+ self.assertEqual(row['dependsOn'], [{'site': CODE_RAW, 'reason': 'unresolved call: computed transfer remains unresolved'}])
285
+ self.assertFalse(r['negativeUsable'])
286
+
287
+ def test_uses_does_not_continue_past_a_callee_that_cannot_return(self):
288
+ # call f; mov [DATA_VA], eax; ret; f: call eax; jmp $
289
+ code = 'e8 06 00 00 00 a3 00 20 40 00 c3 ff d0 eb fe'
290
+ r = report(code, 'uses', query={'offset': DATA_VA, 'width': 4})
291
+ self.assertEqual(r['conditionalAccesses'], [])
292
+ self.assertFalse(r['negativeUsable'])
293
+ with self.assertRaisesRegex(ValueError, 'Positive variable-use control 517 missed'):
294
+ report(code, 'uses', query={'offset': DATA_VA, 'width': 4}, controls=[CODE_RAW + 5])
295
+ # A stop at the callee's return is that return failing, so its caller is not continued either.
296
+ # call f; mov [DATA_VA], eax; ret; f: push 0; ret
297
+ r = report('e8 06 00 00 00 a3 00 20 40 00 c3 6a 00 c3', 'uses', query={'offset': DATA_VA, 'width': 4})
298
+ self.assertEqual(r['conditionalAccesses'], [])
299
+ self.assertFalse(r['negativeUsable'])
300
+
301
+ def test_uses_records_the_limit_of_a_walk_to_a_callee_return(self):
302
+ # call f; mov [DATA_VA], eax; ret; f: mov dx, 0x3c8; out dx, al; nop * 10; ret
303
+ code = 'e8 06 00 00 00 a3 00 20 40 00 c3 66 ba c8 03 ee' + ' 90' * 10 + ' c3'
304
+ for limit, rows in ((100, [5]), (6, [])):
305
+ with self.subTest(limit=limit):
306
+ r = report(code, 'uses', query={'offset': DATA_VA, 'width': 4}, instructionLimit=limit)
307
+ self.assertEqual([e['site'] - CODE_RAW for e in r['conditionalAccesses']], rows)
308
+ self.assertEqual({'site': CODE_RAW + 15, 'reason': 'instruction limit'} in r['gaps'], limit == 6)
309
+ self.assertFalse(r['negativeUsable'])
310
+
197
311
  def test_pop_addresses_its_destination_after_the_stack_pointer_moves(self):
198
312
  # push 1; push 2; push 3; pop dword [esp+4]; pop eax; pop ebx; ret
199
313
  regs = report('6a 01 6a 02 6a 03 8f 44 24 04 58 5b c3')['paths'][0]['registers']
@@ -681,7 +681,7 @@ class GhidraCrossCheckTests(unittest.TestCase):
681
681
  # Ghidra's computed target never becomes an engine edge.
682
682
  self.assertIsNone(r["edges"][2]["target"])
683
683
  self.assertEqual(r["edges"][2]["classification"], "unresolved")
684
- self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 2, "ghidraOnly": 1, "interrupt": 0, "ghidraEndsFunction": 0})
684
+ self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 2, "ghidraOnly": 1, "interrupt": 0, "ghidraEndsFunction": 0, "ghidraContinues": 0})
685
685
  self.assertEqual(check["comparedCallers"], [0, b])
686
686
  self.assertEqual(check["notCompared"]["engineCallers"], [a])
687
687
  self.assertFalse(check["agreed"])
@@ -695,7 +695,7 @@ class GhidraCrossCheckTests(unittest.TestCase):
695
695
  r = run_report(data, cfg, "callees")
696
696
  check = r["ghidraCrossCheck"]
697
697
  self.assertTrue(check["agreed"])
698
- self.assertEqual(check["counts"], {"agreement": 2, "engineOnly": 0, "ghidraOnly": 0, "interrupt": 0, "ghidraEndsFunction": 0})
698
+ self.assertEqual(check["counts"], {"agreement": 2, "engineOnly": 0, "ghidraOnly": 0, "interrupt": 0, "ghidraEndsFunction": 0, "ghidraContinues": 0})
699
699
  # Agreement on an unresolved call leaves the engine's edge unresolved.
700
700
  self.assertEqual(r["edges"][1]["classification"], "unresolved")
701
701
 
@@ -707,7 +707,7 @@ class GhidraCrossCheckTests(unittest.TestCase):
707
707
  cfg = configuration(data, ghidraCallEdges=export)
708
708
  cfg["regions"][0]["entries"] = [0, 6]
709
709
  check = run_report(data, cfg, "callees")["ghidraCrossCheck"]
710
- self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 1, "ghidraOnly": 1, "interrupt": 0, "ghidraEndsFunction": 0})
710
+ self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 1, "ghidraOnly": 1, "interrupt": 0, "ghidraEndsFunction": 0, "ghidraContinues": 0})
711
711
  self.assertFalse(check["agreed"])
712
712
  cfg["controls"] = {"ghidraAgreementSites": [3]}
713
713
  with self.assertRaisesRegex(ValueError, "positive control missed: ghidraAgreementSites"):
@@ -723,7 +723,7 @@ class GhidraCrossCheckTests(unittest.TestCase):
723
723
  r = run_report(data, cfg, "callees")
724
724
  check = r["ghidraCrossCheck"]
725
725
  self.assertTrue(check["agreed"])
726
- self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 0, "ghidraOnly": 0, "interrupt": 2, "ghidraEndsFunction": 0})
726
+ self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 0, "ghidraOnly": 0, "interrupt": 2, "ghidraEndsFunction": 0, "ghidraContinues": 0})
727
727
  rows = [e for e in check["edges"] if e["result"] == "interrupt"]
728
728
  self.assertEqual([(e["site"], e["ghidraFallsThrough"]) for e in rows], [(0, True), (2, True)])
729
729
  self.assertEqual([e["site"] for e in r["edges"]], [3])
@@ -748,7 +748,7 @@ class GhidraCrossCheckTests(unittest.TestCase):
748
748
  data = bytes.fromhex("cc c3")
749
749
  cfg = configuration(data, ghidraCallEdges=ghidra_export(data, {0: [(0, None, "COMPUTED_CALL_TERMINATOR")]}))
750
750
  check = run_report(data, cfg, "callees")["ghidraCrossCheck"]
751
- self.assertEqual(check["counts"], {"agreement": 0, "engineOnly": 0, "ghidraOnly": 0, "interrupt": 1, "ghidraEndsFunction": 1})
751
+ self.assertEqual(check["counts"], {"agreement": 0, "engineOnly": 0, "ghidraOnly": 0, "interrupt": 1, "ghidraEndsFunction": 1, "ghidraContinues": 0})
752
752
  self.assertFalse(check["agreed"])
753
753
 
754
754
  def test_the_exported_falls_through_decides_an_interrupt_over_the_flow_name(self):
@@ -790,7 +790,7 @@ class GhidraCrossCheckTests(unittest.TestCase):
790
790
  self.assertEqual(row["result"], "agreement")
791
791
  self.assertEqual((row["ghidraFallsThrough"], row["ghidraFallsThroughBasis"]), (falls_through, "fallsThrough"))
792
792
  self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 0, "ghidraOnly": 0, "interrupt": 0,
793
- "ghidraEndsFunction": 0 if falls_through else 1})
793
+ "ghidraEndsFunction": 0 if falls_through else 1, "ghidraContinues": 0})
794
794
  self.assertEqual(check["agreed"], agreed)
795
795
  # A call Ghidra ends the function at is no agreement site.
796
796
  cfg["controls"] = {"ghidraAgreementSites": [0]}
@@ -814,19 +814,99 @@ class GhidraCrossCheckTests(unittest.TestCase):
814
814
  self.assertEqual(check["counts"]["ghidraEndsFunction"], 0 if falls_through else 1)
815
815
  self.assertEqual(check["agreed"], falls_through)
816
816
 
817
- def test_an_agreed_tail_transfer_keeps_agreed_without_a_falls_through(self):
818
- # jmp 3; ret; ret. Neither analysis continues past an unconditional jump into another function.
817
+ def test_a_jmp_tail_transfer_ghidra_continues_past_counts_against_agreed(self):
818
+ # jmp 3; ret; ret. The engine stops at the unconditional jump into the entry at 3. Ghidra stops there too,
819
+ # unless a user gave the jump a fall-through.
819
820
  data = bytes.fromhex("eb 01 c3 c3")
820
- cfg = configuration(data, ghidraCallEdges=ghidra_export(data, {0: [(0, 3, "UNCONDITIONAL_JUMP", False)], 3: []}),
821
+ for falls_through in (False, True):
822
+ with self.subTest(fallsThrough=falls_through):
823
+ cfg = configuration(data, ghidraCallEdges=ghidra_export(data, {0: [(0, 3, "UNCONDITIONAL_JUMP", falls_through)], 3: []}))
824
+ cfg["regions"][0]["entries"] = [0, 3]
825
+ check = run_report(data, cfg, "callees")["ghidraCrossCheck"]
826
+ [row] = check["edges"]
827
+ self.assertEqual(row["result"], "agreement")
828
+ self.assertEqual((row["ghidraFallsThrough"], row["ghidraFallsThroughBasis"]), (falls_through, "fallsThrough"))
829
+ self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 0, "ghidraOnly": 0, "interrupt": 0,
830
+ "ghidraEndsFunction": 0, "ghidraContinues": 1 if falls_through else 0})
831
+ self.assertEqual(check["agreed"], not falls_through)
832
+ # A jump Ghidra continues past is no agreement site.
833
+ cfg["controls"] = {"ghidraAgreementSites": [0]}
834
+ if falls_through:
835
+ with self.assertRaisesRegex(ValueError, "positive control missed: ghidraAgreementSites"):
836
+ run_report(data, cfg, "callees")
837
+ else:
838
+ run_report(data, cfg, "callees")
839
+
840
+ def test_a_jmp_in_an_export_without_falls_through_reads_the_flow_name(self):
841
+ # An unconditional jump's flow never falls through, so an export from an older copy of the script agrees at the
842
+ # jump. A user's fall-through override on it goes unseen.
843
+ data = bytes.fromhex("eb 01 c3 c3")
844
+ cfg = configuration(data, ghidraCallEdges=ghidra_export(data, {0: [(0, 3, "UNCONDITIONAL_JUMP")], 3: []}),
821
845
  controls={"ghidraAgreementSites": [0]})
822
846
  cfg["regions"][0]["entries"] = [0, 3]
823
847
  check = run_report(data, cfg, "callees")["ghidraCrossCheck"]
824
848
  [row] = check["edges"]
825
- self.assertEqual(row["result"], "agreement")
826
- self.assertNotIn("ghidraFallsThrough", row)
827
- self.assertEqual(check["counts"]["ghidraEndsFunction"], 0)
849
+ self.assertEqual((row["ghidraFallsThrough"], row["ghidraFallsThroughBasis"]), (False, "flowName"))
850
+ self.assertEqual(check["counts"]["ghidraContinues"], 0)
828
851
  self.assertTrue(check["agreed"])
829
852
 
853
+ def test_a_conditional_flow_name_falls_through_as_ghidra_defines_it(self):
854
+ # je 3; ret; ret. The engine reads on past the conditional jump. Ghidra's CONDITIONAL_TERMINATOR has a
855
+ # fall-through, while its CONDITIONAL_CALL_TERMINATOR does not, although both names contain TERMINATOR.
856
+ data = bytes.fromhex("74 01 c3 c3")
857
+ for flow, falls_through in (("CONDITIONAL_TERMINATOR", True), ("CONDITIONAL_CALL_TERMINATOR", False)):
858
+ with self.subTest(flow=flow):
859
+ cfg = configuration(data, ghidraCallEdges=ghidra_export(data, {0: [(0, 3, flow)], 3: []}))
860
+ cfg["regions"][0]["entries"] = [0, 3]
861
+ check = run_report(data, cfg, "callees")["ghidraCrossCheck"]
862
+ [row] = check["edges"]
863
+ self.assertEqual((row["ghidraFallsThrough"], row["ghidraFallsThroughBasis"]), (falls_through, "flowName"))
864
+ self.assertEqual(check["counts"]["ghidraEndsFunction"], 0 if falls_through else 1)
865
+ self.assertEqual(check["agreed"], falls_through)
866
+
867
+ def test_a_ghidra_only_call_carries_where_ghidra_ends_the_function(self):
868
+ # call 4; ret; ret; ret. The engine resolves the call to 4 and reads on to the ret after it. Ghidra resolves it
869
+ # to 5, and ends the function there when it treats that callee as non-returning.
870
+ data = bytes.fromhex("e8 01 00 c3 c3 c3")
871
+ for flow, falls_through in (("UNCONDITIONAL_CALL", True), ("CALL_TERMINATOR", False)):
872
+ with self.subTest(flow=flow):
873
+ cfg = configuration(data, ghidraCallEdges=ghidra_export(data, {0: [(0, 5, flow, falls_through)], 4: [], 5: []}))
874
+ cfg["regions"][0]["entries"] = [0, 4, 5]
875
+ check = run_report(data, cfg, "callees")["ghidraCrossCheck"]
876
+ [ours, theirs] = check["edges"]
877
+ self.assertEqual((ours["result"], ours["target"]), ("engineOnly", 4))
878
+ self.assertNotIn("ghidraFallsThrough", ours)
879
+ self.assertEqual((theirs["result"], theirs["target"], theirs["engineEdge"]), ("ghidraOnly", 5, ours["engineEdge"]))
880
+ self.assertEqual((theirs["ghidraFallsThrough"], theirs["ghidraFallsThroughBasis"]), (falls_through, "fallsThrough"))
881
+ self.assertEqual(check["counts"], {"agreement": 0, "engineOnly": 1, "ghidraOnly": 1, "interrupt": 0,
882
+ "ghidraEndsFunction": 0 if falls_through else 1, "ghidraContinues": 0})
883
+ self.assertFalse(check["agreed"])
884
+
885
+ def test_a_ghidra_only_edge_at_an_instruction_the_engine_did_not_read_has_no_falls_through(self):
886
+ # ret; ret. Ghidra claims a call at 1, past the end of the engine's body for the entry at 0, so the engine has no
887
+ # reading there to compare Ghidra's fall-through with.
888
+ data = bytes.fromhex("c3 c3")
889
+ cfg = configuration(data, ghidraCallEdges=ghidra_export(data, {0: [(1, None, "COMPUTED_CALL_TERMINATOR", False)]}))
890
+ check = run_report(data, cfg, "callees")["ghidraCrossCheck"]
891
+ [row] = check["edges"]
892
+ self.assertEqual(row["result"], "ghidraOnly")
893
+ self.assertNotIn("ghidraFallsThrough", row)
894
+ self.assertEqual((check["counts"]["ghidraEndsFunction"], check["counts"]["ghidraContinues"]), (0, 0))
895
+ self.assertFalse(check["agreed"])
896
+
897
+ def test_a_ghidra_only_edge_at_a_transfer_outside_the_frame_model_has_no_falls_through(self):
898
+ # o32 call 7; ret; ret. The engine stops at the operand-size call with a gap and decides nothing about
899
+ # fall-through there, so Ghidra's terminator is not compared.
900
+ data = bytes.fromhex("66 e8 01 00 00 00 c3 c3")
901
+ cfg = configuration(data, ghidraCallEdges=ghidra_export(data, {0: [(0, 7, "CALL_TERMINATOR", False)], 7: []}))
902
+ cfg["regions"][0]["entries"] = [0, 7]
903
+ check = run_report(data, cfg, "callees")["ghidraCrossCheck"]
904
+ [row] = check["edges"]
905
+ self.assertEqual(row["result"], "ghidraOnly")
906
+ self.assertNotIn("ghidraFallsThrough", row)
907
+ self.assertEqual((check["counts"]["ghidraEndsFunction"], check["counts"]["ghidraContinues"]), (0, 0))
908
+ self.assertFalse(check["agreed"])
909
+
830
910
  def test_a_conditional_tail_transfer_ghidra_ends_the_function_at_counts_against_agreed(self):
831
911
  # je 3; ret; ret. The engine reads on past the conditional jump into the entry at 3; Ghidra ends the function
832
912
  # there when a user cleared the jump's fall-through.
@@ -869,7 +949,7 @@ class GhidraCrossCheckTests(unittest.TestCase):
869
949
  cfg["regions"][0]["entries"] = [0, 6]
870
950
  r = run_report(data, cfg, "callees")
871
951
  check = r["ghidraCrossCheck"]
872
- self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 0, "ghidraOnly": 0, "interrupt": 0, "ghidraEndsFunction": 0})
952
+ self.assertEqual(check["counts"], {"agreement": 1, "engineOnly": 0, "ghidraOnly": 0, "interrupt": 0, "ghidraEndsFunction": 0, "ghidraContinues": 0})
873
953
  self.assertEqual(check["notCompared"]["omittedEngineRoutes"], [r["omittedRoutes"][0]["id"]])
874
954
  self.assertFalse(check["agreed"])
875
955