scientific-method-engine 19.8.0__tar.gz → 19.9.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/PKG-INFO +1 -1
  2. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/pyproject.toml +1 -1
  3. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/cli.py +1 -1
  4. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/inventory.py +6 -3
  5. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/reach.py +12 -6
  6. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/reports.py +3 -0
  7. scientific_method_engine-19.9.0/src/scientific_method_engine/x86/scan_index.py +181 -0
  8. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/trace.py +28 -7
  9. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_inventory_check.py +23 -0
  10. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_reach.py +80 -5
  11. scientific_method_engine-19.9.0/tests/test_scan_index.py +163 -0
  12. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/.gitignore +0 -0
  13. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/LICENSE +0 -0
  14. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/README.md +0 -0
  15. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/__init__.py +0 -0
  16. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/__main__.py +0 -0
  17. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  18. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  19. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  20. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  21. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  22. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  23. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  24. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  25. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  26. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  27. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  28. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  29. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  30. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  31. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  32. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  33. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  34. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  35. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  36. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  37. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  38. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  39. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  40. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  41. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  42. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  43. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
  44. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  45. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  46. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  47. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  48. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
  49. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
  50. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
  51. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
  52. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  53. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
  54. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/derived_counts.py +0 -0
  55. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  56. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  57. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/image.py +0 -0
  58. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/loops.py +0 -0
  59. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/machine.py +0 -0
  60. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
  61. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/pcode.py +0 -0
  62. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
  63. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/pe.py +0 -0
  64. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/ranges.py +0 -0
  65. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/relational.py +0 -0
  66. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  67. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/table_bounds.py +0 -0
  68. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/values.py +0 -0
  69. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/oracle.py +0 -0
  70. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_config_encoding.py +0 -0
  71. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_continuation_budget.py +0 -0
  72. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_derived_counts.py +0 -0
  73. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_dispatch.py +0 -0
  74. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_effect_order.py +0 -0
  75. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_enter_frames.py +0 -0
  76. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_entry_frame.py +0 -0
  77. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_indirect_far_transfers.py +0 -0
  78. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_interrupt_models.py +0 -0
  79. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_join_terms.py +0 -0
  80. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_loops.py +0 -0
  81. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_memory_scopes.py +0 -0
  82. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_nested_frame_request.py +0 -0
  83. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_oracle.py +0 -0
  84. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_pe.py +0 -0
  85. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_ranges.py +0 -0
  86. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_relational_controls.py +0 -0
  87. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_return_checks.py +0 -0
  88. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_successor_rule.py +0 -0
  89. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_table_bounds.py +0 -0
  90. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_table_continuations.py +0 -0
  91. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_term_limit.py +0 -0
  92. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_upstream_gaps.py +0 -0
  93. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_volatile_memory.py +0 -0
  94. {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_x86.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 19.8.0
3
+ Version: 19.9.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "19.8.0"
8
+ version = "19.9.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -13,7 +13,7 @@ PREPARED_CONFIG_LIMIT = 16 * 1024 * 1024
13
13
  # The header names the decoder and instruction semantics that actually ran, not the pins in pyproject.toml.
14
14
  DECODER = f"capstone {CAPSTONE_VERSION or 'without distribution metadata'}"
15
15
  INSTRUCTION_SEMANTICS = f"pypcode {pypcode.__version__} (Ghidra SLEIGH x86)"
16
- USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bounds|owner|callees|reach|trace|uses|arguments|"
16
+ USAGE = ("Usage: scientific-method-engine <scan-index|operand|operand-candidates|target|bounds|owner|callees|reach|trace|uses|arguments|"
17
17
  "effects|returns|memory|incoming|inventory-check|call-order|guards|allocation|dispatch> <config.json|->\n"
18
18
  "effects includes ordered path writes/calls and local restoration witnesses; transactionality remains unestablished.\n"
19
19
  "callees compares its edges with an ExportCallEdges.java export given as ghidraCallEdges;\n"
@@ -239,15 +239,18 @@ def _no_return_check(image, routines, declared, limit):
239
239
  ``routines`` are the declared routines, or the targets of the declared calls. The walk starts at
240
240
  them and continues past every interrupt except a declared one, as ``reach`` reads them, so a
241
241
  routine that ends in an interrupt that returns is contradicted by what follows it. Calls are
242
- stepped over at their return sites, except calls to declared routines and declared calls.
242
+ stepped over at their return sites with their targets resolved, so a call to a declared routine
243
+ or at a declared site still ends its branch, and no callee other than a declared routine is
244
+ decoded or spends ``limit``: the walk reads only the routines' and the declared routines' own paths.
243
245
  A routine the walk did not establish, one that does not decode or that a rejected overlap removed,
244
246
  has no return sites and is not in ``read``.
245
247
  """
246
248
  from .reach import returns_from, successor_graph
247
249
  if not routines:
248
250
  return {}, {}, False
249
- seen, gaps, _, _, _ = walk(image, sorted(routines), limit, follow_interrupts=True, **declared)
250
- graph = successor_graph(image, seen, follow_interrupts=True, **declared)
251
+ seen, gaps, _, _, _ = walk(image, sorted(routines), limit, follow_interrupts=True, resolve_over_calls=True,
252
+ **declared)
253
+ graph = successor_graph(image, seen, follow_interrupts=True, resolve_over_calls=True, **declared)
251
254
  return ({at: returns_from(graph, seen, set(), at) for at in routines}, seen,
252
255
  any(g["reason"] == LIMIT_REASON for g in gaps))
253
256
 
@@ -1046,10 +1046,13 @@ def reach(image, config):
1046
1046
  or any(at in seen or at in contested or at in overlapping for at in stop_sites))
1047
1047
  if stop_met and checked:
1048
1048
  # A stop cuts the query and says nothing about the build, so the return check reads the reached
1049
- # declared routines without the stops: code a stop hid can still contradict a declaration.
1049
+ # declared routines without the stops: code a stop hid can still contradict a declaration. It
1050
+ # steps over calls with their targets resolved, entering only a declared routine, so only the
1051
+ # declared routines' own paths are decoded and spend the instruction limit; returns_from skips
1052
+ # call edges either way.
1050
1053
  read, check_gaps, _, _, check_contested = walk(image, sorted(at for at in checked if at in distance),
1051
1054
  instruction_limit, follow_interrupts=True,
1052
- stops=frozenset(leaves), **declared)
1055
+ stops=frozenset(leaves), resolve_over_calls=True, **declared)
1053
1056
  check_stopped = any(g["reason"] == LIMIT_REASON for g in check_gaps)
1054
1057
  # What the check could not read past a stop leaves its declarations unchecked there, as a gap of the
1055
1058
  # query's own walk would; the rows the query's walk already holds are not repeated.
@@ -1059,7 +1062,8 @@ def reach(image, config):
1059
1062
  and g["site"] not in check_contested]
1060
1063
  + [{"site": at, "reason": CHECK_CONTESTED} for at in check_contested if at not in contested],
1061
1064
  key=lambda g: (g["site"], g["reason"]))
1062
- check_graph = successor_graph(image, read, kept=frozenset(leaves), follow_interrupts=True, **declared)
1065
+ check_graph = successor_graph(image, read, kept=frozenset(leaves), follow_interrupts=True,
1066
+ resolve_over_calls=True, **declared)
1063
1067
  returns = {at: returns_from(check_graph, read, returning_leaves, at) for at in checked}
1064
1068
  no_return = no_return_rows(no_return_routines, no_return_interrupts, returns, ended_sites, seen, distance, read,
1065
1069
  no_return_sites, site_targets)
@@ -1157,9 +1161,11 @@ def reach(image, config):
1157
1161
  "A stop narrows the question and claims nothing about the build: the walk reads no "
1158
1162
  "further at a stop site, nor into a stop routine or past a call to it, so a target "
1159
1163
  "reached only past a stop is not reached, and negativeUsable answers the narrowed "
1160
- "question. The noReturn return check reads past the stops, with a walk of its own, and "
1161
- "returnCheckGaps lists what that walk could not read that the query's walk did not "
1162
- "record; any row keeps negativeUsable false, as a gap would. "
1164
+ "question. The noReturn return check reads past the stops, with a walk of its own that "
1165
+ "steps over calls at their return sites after resolving their targets, so it decodes "
1166
+ "only the declared routines' own paths; returnCheckGaps lists what that walk could not "
1167
+ "read on those paths that the query's walk did not record, and any row keeps "
1168
+ "negativeUsable false, as a gap would. "
1163
1169
  "decodedRanges, when asked for, covers the instructions the walk decoded; no range "
1164
1170
  "starts at a leaf, a stop routine or a contested, unresolved overlapping or undecodable start, though "
1165
1171
  "one can lie inside a decoded instruction. A site inside a range lies in a decoded "
@@ -1884,6 +1884,9 @@ def _run_report(image, config, command):
1884
1884
  raise ValueError("indirectCalls applies only to reach and inventory-check")
1885
1885
  if command == "operand":
1886
1886
  return operand_provenance(image, config)
1887
+ if command == "scan-index":
1888
+ from .scan_index import scan_index
1889
+ return scan_index(image, config)
1887
1890
  if command == "target":
1888
1891
  return call_target_report(image, config)
1889
1892
  if command == "bounds":
@@ -0,0 +1,181 @@
1
+ """Reusable raw caller/operand candidates. No reachability or instruction-start claims."""
2
+ import hashlib
3
+ from contextlib import closing
4
+ import json
5
+ import os
6
+ from pathlib import Path
7
+ import sqlite3
8
+ import tempfile
9
+
10
+ from capstone import CS_GRP_CALL
11
+ from capstone.x86 import X86_OP_IMM, X86_OP_MEM
12
+ import xxhash
13
+
14
+ from .image import CAPSTONE_VERSION, integer, scan_limit
15
+ from .reports import _relative_transfer
16
+ from .trace import call_target, unsupported_transfer
17
+
18
+ SCHEMA = 2
19
+ MAX_INDEX_BYTES = 256 * 1024 * 1024
20
+ INSERT_BATCH = 10000
21
+
22
+
23
+ def _identity(image):
24
+ # Query, output location and source pathname do not change the decoded corpus.
25
+ config = {k: v for k, v in image.config.items() if k not in ("query", "scanIndex", "source")}
26
+ # The digest keeps the stored metadata and every report small however large the prepared config is.
27
+ config_digest = hashlib.sha256(json.dumps(config, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
28
+ sources = hashlib.sha256()
29
+ root = Path(__file__).resolve().parent.parent
30
+ for path in sorted(root.rglob("*.py")):
31
+ sources.update(path.relative_to(root).as_posix().encode())
32
+ sources.update(path.read_bytes())
33
+ return {"schema": SCHEMA, "xxh3": xxhash.xxh3_128_hexdigest(image.data),
34
+ "size": len(image.data), "capstone": CAPSTONE_VERSION,
35
+ "engineSources": sources.hexdigest(), "configSha256": config_digest}
36
+
37
+
38
+ def _build(image, destination, identity):
39
+ handle, temporary = tempfile.mkstemp(prefix="scan-index-", suffix=".partial", dir=destination.parent)
40
+ os.close(handle)
41
+ try:
42
+ with closing(sqlite3.connect(temporary)) as db, db:
43
+ # Bound disk use, including the final query index. No partial index is published.
44
+ db.execute("PRAGMA page_size=4096")
45
+ db.execute(f"PRAGMA max_page_count={MAX_INDEX_BYTES // 4096}")
46
+ db.execute("CREATE TABLE candidates (site INTEGER, size INTEGER, kind TEXT, value INTEGER)")
47
+ db.execute("CREATE TABLE metadata (value TEXT)")
48
+ db.execute("CREATE TABLE undecoded (start INTEGER, end INTEGER)")
49
+ db.execute("CREATE TABLE unresolved_calls (site INTEGER, size INTEGER)")
50
+ undecoded = unresolved = rows = 0
51
+ pending = []
52
+ for region in image.regions:
53
+ gap = None
54
+ for site in range(region["start"], region["end"]):
55
+ ins = image.decode(site)
56
+ # The one-shot decoder cache is unnecessary during a whole-corpus scan.
57
+ image.cache.pop(site, None)
58
+ if ins is None:
59
+ undecoded += 1
60
+ gap = site if gap is None else gap
61
+ continue
62
+ if gap is not None:
63
+ db.execute("INSERT INTO undecoded VALUES (?, ?)", (gap, site))
64
+ gap = None
65
+ found = set()
66
+ for operand in ins.operands:
67
+ # Only encoded literals, as operand-candidates reads them: implicit forms (SHL r,1;
68
+ # [BX]) and relative branch targets are not immediates or displacements.
69
+ if operand.type == X86_OP_IMM and ins.imm_size and not _relative_transfer(ins):
70
+ found.add(("immediate", operand.imm))
71
+ elif operand.type == X86_OP_MEM and ins.disp_size:
72
+ found.add(("displacement", operand.mem.disp))
73
+ if ins.group(CS_GRP_CALL):
74
+ # An operand-size-overridden call is outside the frame model, as direct_calls reports it.
75
+ target = None if unsupported_transfer(image, ins) else call_target(image, site, ins)[0]
76
+ if target is None:
77
+ unresolved += 1
78
+ db.execute("INSERT INTO unresolved_calls VALUES (?, ?)", (site, ins.size))
79
+ else:
80
+ found.add(("call", target))
81
+ pending.extend((site, ins.size, kind, value) for kind, value in sorted(found))
82
+ rows += len(found)
83
+ if len(pending) >= INSERT_BATCH:
84
+ db.executemany("INSERT INTO candidates VALUES (?, ?, ?, ?)", pending)
85
+ pending.clear()
86
+ if gap is not None:
87
+ db.execute("INSERT INTO undecoded VALUES (?, ?)", (gap, region["end"]))
88
+ db.executemany("INSERT INTO candidates VALUES (?, ?, ?, ?)", pending)
89
+ db.execute("CREATE INDEX lookup ON candidates (kind, value, site)")
90
+ metadata = {"identity": identity, "scannedBytes": image.code_bytes,
91
+ "undecodedOffsets": undecoded, "unresolvedCallCandidates": unresolved,
92
+ "candidateRows": rows}
93
+ db.execute("INSERT INTO metadata VALUES (?)", (json.dumps(metadata, sort_keys=True),))
94
+ # A concurrent builder of the same identity produces the same corpus. On Windows the replace
95
+ # fails while another run has that corpus open, which then already holds the same rows.
96
+ try:
97
+ os.replace(temporary, destination)
98
+ except PermissionError:
99
+ if not destination.exists():
100
+ raise
101
+ finally:
102
+ if os.path.exists(temporary):
103
+ os.unlink(temporary)
104
+
105
+
106
+ def scan_index(image, config):
107
+ """Query a content-keyed local SQLite index of every-offset decode candidates.
108
+
109
+ ``scanIndex.directory`` must be an absolute directory.
110
+ ``query`` names ``kind`` (immediate, displacement, call), integer ``value``,
111
+ ``limit`` (1..10000, default 1000) and optional matching-site ``controls``.
112
+ A call value is a mapped source-file offset, not a loaded address. Every
113
+ result remains a raw candidate, including prefix and interior decodes.
114
+ ``undecodedRanges`` (half-open) and ``unresolvedCalls`` list where the scan
115
+ could not see, up to ``limit`` rows each.
116
+ Source verification and MZ/FBOV preparation stay with the normal reader.
117
+ """
118
+ options, query = config.get("scanIndex"), config.get("query")
119
+ if not isinstance(options, dict) or set(options) != {"directory"}:
120
+ raise ValueError("scanIndex needs only directory")
121
+ directory = options["directory"]
122
+ if not isinstance(directory, str) or not Path(directory).is_absolute():
123
+ raise ValueError("scanIndex.directory must be an absolute path under GAME_DIR")
124
+ if not isinstance(query, dict) or set(query) - {"kind", "value", "limit", "controls"}:
125
+ raise ValueError("scan-index query needs kind, value, optional limit and controls")
126
+ kind = query.get("kind")
127
+ if kind not in ("immediate", "displacement", "call"):
128
+ raise ValueError("scan-index kind must be immediate, displacement or call")
129
+ value = integer(query.get("value"), -(1 << 63), (1 << 63) - 1, "query value")
130
+ limit = integer(query.get("limit", 1000), 1, 10000, "query limit")
131
+ controls = query.get("controls", [])
132
+ if not isinstance(controls, list) or len(controls) > 128:
133
+ raise ValueError("query controls must list at most 128 matching source offsets")
134
+ for site in controls:
135
+ integer(site, 0, len(image.data) - 1, "control site")
136
+ budget = scan_limit(image, config.get("scanLimit", 1048576))
137
+ if image.code_bytes > budget:
138
+ raise ValueError("scan-index scanLimit reached; no complete index produced")
139
+ identity = _identity(image)
140
+ key = hashlib.sha256(json.dumps(identity, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
141
+ folder = Path(directory)
142
+ folder.mkdir(parents=True, exist_ok=True)
143
+ destination = folder / (key + ".sqlite")
144
+ hit = destination.exists()
145
+ if not hit:
146
+ try:
147
+ _build(image, destination, identity)
148
+ except sqlite3.Error as error:
149
+ raise ValueError(f"scan-index build failed and published nothing: {error}") from None
150
+ try:
151
+ if destination.stat().st_size > MAX_INDEX_BYTES:
152
+ raise ValueError("scan-index exceeds the 256 MiB cache limit")
153
+ with closing(sqlite3.connect(destination.as_uri() + "?mode=ro", uri=True)) as db:
154
+ raw = db.execute("SELECT value FROM metadata").fetchall()
155
+ if len(raw) != 1:
156
+ raise ValueError("scan-index metadata is invalid; remove this local cache and rebuild")
157
+ metadata = json.loads(raw[0][0])
158
+ if not isinstance(metadata, dict) or metadata.get("identity") != identity:
159
+ raise ValueError("scan-index identity differs; remove this local cache and rebuild")
160
+ missing = [site for site in controls if db.execute(
161
+ "SELECT 1 FROM candidates WHERE kind=? AND value=? AND site=? LIMIT 1",
162
+ (kind, value, site)).fetchone() is None]
163
+ if missing:
164
+ raise ValueError(f"scan-index matching-site controls missed: {missing}")
165
+ total = db.execute("SELECT count(*) FROM candidates WHERE kind=? AND value=?", (kind, value)).fetchone()[0]
166
+ rows = db.execute("SELECT site, size FROM candidates WHERE kind=? AND value=? ORDER BY site LIMIT ?",
167
+ (kind, value, limit)).fetchall()
168
+ # A negative result is only as complete as the offsets that decoded and the calls that resolved.
169
+ gaps = db.execute("SELECT start, end FROM undecoded ORDER BY start LIMIT ?", (limit + 1,)).fetchall()
170
+ calls = db.execute("SELECT site, size FROM unresolved_calls ORDER BY site LIMIT ?", (limit + 1,)).fetchall()
171
+ except (sqlite3.Error, json.JSONDecodeError) as error:
172
+ raise ValueError(f"scan-index cache failed; remove this local cache and rebuild: {error}") from None
173
+ return {"command": "scan-index", "cacheKey": key, "cacheHit": hit,
174
+ "scan": metadata, "query": query, "totalCandidates": total,
175
+ "resultLimitReached": total > limit,
176
+ "rawCandidates": [{"site": site, "size": size} for site, size in rows],
177
+ "undecodedRanges": [{"start": start, "end": end} for start, end in gaps[:limit]],
178
+ "undecodedRangesLimitReached": len(gaps) > limit,
179
+ "unresolvedCalls": [{"site": site, "size": size} for site, size in calls[:limit]],
180
+ "unresolvedCallsLimitReached": len(calls) > limit,
181
+ "meaning": "Every-offset decode candidates only; no entry-path qualification or absence proof."}
@@ -181,7 +181,7 @@ class Step(NamedTuple):
181
181
 
182
182
  def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, follow_interrupts=False,
183
183
  modeled_interrupts=frozenset(), no_return_calls=frozenset(), no_return_interrupts=frozenset(),
184
- indirect_calls=None, no_return_call_sites=frozenset()):
184
+ indirect_calls=None, no_return_call_sites=frozenset(), resolve_over_calls=False):
185
185
  """The successors of ``ins`` decoded at ``at``, under the rule ``walk`` and the ``uses`` caller continuation share.
186
186
 
187
187
  An unsupported transfer encoding, a return, an unconditional jump without a resolved target,
@@ -190,7 +190,12 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
190
190
  conditional jump or loop continues at its resolved target and at the next instruction. A declared
191
191
  indirect jump continues at its table rows only. A call continues at its resolved target and at
192
192
  its return site; with ``step_over_calls`` it continues at its return site only and its target
193
- is neither resolved nor reported. A call whose resolved target is in ``no_return_calls``, and an
193
+ is neither resolved nor reported. With ``resolve_over_calls`` it also continues at its return
194
+ site only, but the target is resolved first, so the declarations below still end their branches:
195
+ a call to a routine in ``no_return_calls`` ends the branch, an unresolved target still records
196
+ its gap, and a declared indirect call keeps its declaration's rule. Only a target in
197
+ ``no_return_calls`` is entered and reported as an edge, since a declared routine's own paths are
198
+ ones the walk reads; no other callee is. A call whose resolved target is in ``no_return_calls``, and an
194
199
  interrupt whose site is in ``no_return_interrupts``, have no return site: the query declared that
195
200
  they never come back. A call whose site is in ``no_return_call_sites`` has no return site either,
196
201
  whatever its target: the query declared that this one call never comes back. A call whose site
@@ -215,7 +220,9 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
215
220
  return Step(targets, False, None, gaps, edges, supplied=True)
216
221
  declaration = (indirect_calls or {}).get(at)
217
222
  if declaration is not None:
218
- targets = list(declaration["targets"])
223
+ # Under resolve_over_calls only a declared noReturn routine is entered, and successor_kinds counts the
224
+ # target edges as the call successors, so a target that is not entered gets no edge either.
225
+ targets = [t for t in declaration["targets"] if not resolve_over_calls or t in no_return_calls]
219
226
  edges = [{"site": at, "target": target, "kind": m,
220
227
  "provenance": {"encoding": "declared indirect call", "evidence": declaration["evidence"]}}
221
228
  for target in targets]
@@ -232,7 +239,17 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
232
239
  return Step([], True, None, [], [])
233
240
  successors, gaps, edges = [], [], []
234
241
  calls = m in ("call", "lcall")
235
- if (calls and not step_over_calls) or m == "ljmp" or m.startswith(("j", "loop")):
242
+ if calls and resolve_over_calls:
243
+ # The target is resolved to apply the declarations. A declared noReturn routine is entered, since its own
244
+ # paths are ones the check reads; any other callee is neither entered nor reported as an edge.
245
+ target, provenance = call_target(image, at, ins)
246
+ if target is None:
247
+ edges.append({"site": at, "target": None, "kind": m, "provenance": provenance})
248
+ gaps.append({"site": at, "reason": provenance.get("reason", "target outside declared regions")})
249
+ elif target in no_return_calls:
250
+ edges.append({"site": at, "target": target, "kind": m, "provenance": provenance})
251
+ return Step([target], False, None, gaps, edges)
252
+ elif (calls and not step_over_calls) or m == "ljmp" or m.startswith(("j", "loop")):
236
253
  target, provenance = call_target(image, at, ins)
237
254
  edges.append({"site": at, "target": target, "kind": m, "provenance": provenance})
238
255
  if target is None:
@@ -261,7 +278,7 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
261
278
 
262
279
  def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts=False, stops=frozenset(),
263
280
  modeled_interrupts=frozenset(), no_return_calls=frozenset(), no_return_interrupts=frozenset(),
264
- indirect_calls=None, no_return_call_sites=frozenset(), ends=frozenset()):
281
+ indirect_calls=None, no_return_call_sites=frozenset(), ends=frozenset(), resolve_over_calls=False):
265
282
  """Decode the CFG reached from ``entries`` and check its instruction boundaries.
266
283
 
267
284
  A port access in the flat model records a gap and ends that branch, as ``trace`` stops there.
@@ -274,7 +291,11 @@ def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts
274
291
  decoded and its boundary checked, and the walk follows none of its successors.
275
292
  ``no_return_calls`` and ``no_return_interrupts`` end the branch at a call to one of those routines and at one of those
276
293
  interrupt sites, ``no_return_call_sites`` at a call at one of those sites, and ``indirect_calls``
277
- continues a declared indirect call at its declared targets, as ``cfg_step`` describes. A
294
+ continues a declared indirect call at its declared targets, as ``cfg_step`` describes.
295
+ ``resolve_over_calls`` steps over every call at its return site after resolving its target, so
296
+ those declarations still end their branches but no callee is entered other than a routine in
297
+ ``no_return_calls``: the walk reads only the entries' and the declared routines' own paths, and
298
+ only they spend ``limit``. A
278
299
  declared table whose count was derived from the code, and that the walk reads, gets a gap
279
300
  (``derived_counts.entry_gaps``) when an edge the walk read enters the routine the count was
280
301
  derived over other than at its start.
@@ -305,7 +326,7 @@ def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts
305
326
  step = cfg_step(image, at, ins, follow_flat_ports, follow_interrupts=follow_interrupts,
306
327
  modeled_interrupts=modeled_interrupts, no_return_calls=no_return_calls,
307
328
  no_return_interrupts=no_return_interrupts, indirect_calls=indirect_calls,
308
- no_return_call_sites=no_return_call_sites)
329
+ no_return_call_sites=no_return_call_sites, resolve_over_calls=resolve_over_calls)
309
330
  successors[at] = step.successors
310
331
  gaps.extend(step.gaps)
311
332
  edges.extend(step.edges)
@@ -431,6 +431,29 @@ class InventoryCheckTests(unittest.TestCase):
431
431
  self.assertIn("The walk that checks the noReturn routines stopped at its instruction limit", stopped["summary"])
432
432
  self.assertFalse(r["noReturnCheckLimitReached"])
433
433
 
434
+ def test_the_no_return_check_reads_only_the_routines_own_paths(self):
435
+ # 0000 ret is the whole entry path. The declared routine at 0001 calls 0008, whose seven
436
+ # instructions a check that entered them would spend a limit of four on; its own path holds two.
437
+ data = bytes.fromhex("c3" "e80400" "ebfe" "9090" + "90" * 6 + "c3")
438
+ inventory = "start\tsize\n1000:0000\t1\n1000:0001\t7\n1000:0008\t7\n"
439
+ exit_routine = [{"routine": 1, "reason": "synthetic exit"}]
440
+ r = self.check(inventory, data=data, noReturn=exit_routine, instructionLimit=4)
441
+ self.assertFalse(r["noReturnCheckLimitReached"])
442
+ row = r["noReturn"][0]
443
+ self.assertEqual((row["returnSites"], row["contradicted"], row["read"], row["reached"]), ([], False, True, False))
444
+ # A return on the routine's own path past the stepped-over call still contradicts the declaration.
445
+ returning = bytes.fromhex("c3" "e80400" "c3" "909090" + "90" * 6 + "c3")
446
+ r = self.check(inventory, data=returning, noReturn=exit_routine, instructionLimit=4)
447
+ self.assertFalse(r["noReturnCheckLimitReached"])
448
+ self.assertEqual((r["noReturn"][0]["returnSites"], r["noReturn"][0]["contradicted"]), ([4], True))
449
+ # A declared indirect call on the routine's own path is stepped over too, and the return past it
450
+ # still contradicts the declaration.
451
+ declared_call = bytes.fromhex("c3" "ffd3" "c3" "c3")
452
+ r = self.check("start\tsize\n1000:0000\t1\n1000:0001\t3\n1000:0004\t1\n", data=declared_call,
453
+ noReturn=exit_routine, indirectCalls=[{"site": 1, "exhaustive": True, "targets": [4],
454
+ "evidence": "synthetic: bx holds 0004"}])
455
+ self.assertEqual((r["noReturn"][0]["returnSites"], r["noReturn"][0]["contradicted"]), ([3], True))
456
+
434
457
  def test_a_contested_instruction_holds_no_target(self):
435
458
  # Entries 0 and 1 overlap, so the call at 0003 is contested. The raw call at 000D targets 0004, a byte of
436
459
  # that contested call, and names no holder, since only established instructions count.
@@ -604,15 +604,16 @@ class StopTests(unittest.TestCase):
604
604
  self.assertEqual((r["noReturn"][0]["returnSites"], r["noReturn"][0]["contradicted"]), ([7], True))
605
605
  self.assertTrue(r["noReturn"][0]["read"])
606
606
  self.assertFalse(r["negativeUsable"])
607
- # The check has the walk's instruction limit. Here R spins after its call to S at 0009, so the walk
608
- # reads only the call into R and R's call to S, and the check, which reads S too, stops before its end.
609
- data = bytes.fromhex("e80100" "c3" "e80200" "ebfe" "9090c3")
610
- stop = [{"routine": 9, "reason": CUT}]
607
+ # The check has the walk's instruction limit, spent on the routine's own paths. The call to the
608
+ # stop routine S ends the query's branch, so R's three instructions past it and its spin are read
609
+ # by the check alone, and a limit of three stops the check there.
610
+ data = bytes.fromhex("e80100" "c3" "e80500" "909090" "ebfe" "c3")
611
+ stop = [{"routine": 0x0C, "reason": CUT}]
611
612
  r = run_report(data, config(data, targets=[3], controls=[0], noReturn=no_return, stops=stop), "reach")
612
613
  self.assertEqual((r["noReturn"][0]["contradicted"], r["instructionLimitReached"], r["negativeUsable"]),
613
614
  (False, False, True))
614
615
  r = run_report(data, config(data, targets=[3], controls=[0], noReturn=no_return, stops=stop,
615
- instructionLimit=2), "reach")
616
+ instructionLimit=3), "reach")
616
617
  self.assertEqual(r["counts"]["instructions"], 2)
617
618
  self.assertTrue(r["instructionLimitReached"])
618
619
  self.assertFalse(r["negativeUsable"])
@@ -631,6 +632,80 @@ class StopTests(unittest.TestCase):
631
632
  self.assertEqual([row["site"] for row in r["returnCheckGaps"]], [5])
632
633
  self.assertFalse(r["negativeUsable"])
633
634
 
635
+ def test_the_no_return_check_reads_only_the_routines_own_paths(self):
636
+ # 0000 call R at 0004; 0003 ret. R calls the stop routine S at 000B and spins at 0007. S holds
637
+ # a computed jump nothing resolves, so a check that entered it would gain a returnCheckGaps row.
638
+ data = bytes.fromhex("e80100" "c3" "e80400" "ebfe" "9090" "90" "ffe0" "c3")
639
+ no_return = [{"routine": 4, "reason": "synthetic: declared to end the program"}]
640
+ stops = [{"routine": 0x0B, "reason": CUT}]
641
+ r = run_report(data, config(data, targets=[3], controls=[0], noReturn=no_return, stops=stops), "reach")
642
+ row = r["noReturn"][0]
643
+ self.assertEqual((row["returnSites"], row["contradicted"], row["read"]), ([], False, True))
644
+ self.assertEqual(r["returnCheckGaps"], [])
645
+ self.assertFalse(r["instructionLimitReached"])
646
+ self.assertTrue(r["negativeUsable"])
647
+ # A return on the routine's own path past the stepped-over call still contradicts the declaration.
648
+ returning = bytes.fromhex("e80100" "c3" "e80400" "c3" "909090" "90" "ffe0" "c3")
649
+ r = run_report(returning, config(returning, targets=[3], controls=[0], noReturn=no_return, stops=stops),
650
+ "reach")
651
+ self.assertEqual((r["noReturn"][0]["returnSites"], r["noReturn"][0]["contradicted"]), ([7], True))
652
+ self.assertEqual(r["returnCheckGaps"], [])
653
+ # A call to its own next instruction keeps its return site on the routine's path. The two arms
654
+ # of the branch at 0000 call the stop routine S at 000A and the declared routine R at 000B.
655
+ self_call = bytes.fromhex("7503" "e80500" "e80300" "c3" "90" "c3" "e80000" "c3")
656
+ r = run_report(self_call, config(self_call, targets=[8], controls=[2, 5],
657
+ noReturn=[{"routine": 0x0B, "reason": "synthetic: declared to end the program"}],
658
+ stops=[{"routine": 0x0A, "reason": CUT}]), "reach")
659
+ self.assertEqual((r["noReturn"][0]["returnSites"], r["noReturn"][0]["contradicted"]), ([0x0E], True))
660
+
661
+ def test_a_call_to_another_declared_routine_still_ends_the_checks_branch(self):
662
+ # The two arms of the branch at 0000 call the stop routine S at 000A and R1 at 000B. R1 calls
663
+ # R2 at 000F, both declared, with a return instruction after the call that is on no path of
664
+ # R1's own.
665
+ data = bytes.fromhex("7503" "e80500" "e80300" "c3" "90" "c3" "e80100" "c3" "ebfe")
666
+ no_return = [{"routine": 0x0B, "reason": "synthetic: declared to end the program"},
667
+ {"routine": 0x0F, "reason": "synthetic: declared to end the program"}]
668
+ r = run_report(data, config(data, targets=[8], controls=[2, 5], noReturn=no_return,
669
+ stops=[{"routine": 0x0A, "reason": CUT}]), "reach")
670
+ self.assertEqual([(row["returnSites"], row["contradicted"]) for row in r["noReturn"]],
671
+ [([], False), ([], False)])
672
+ self.assertEqual(r["returnCheckGaps"], [])
673
+ self.assertTrue(r["negativeUsable"])
674
+
675
+ def test_the_check_reads_past_a_stepped_over_declared_indirect_call(self):
676
+ # 0000 call R at 0004; 0003 ret. R calls the stop routine S at 000C, then makes a declared call
677
+ # at 0007 to T at 000D and returns at 0009, which only the check reads.
678
+ data = bytes.fromhex("e80100" "c3" "e80500" "ffd3" "c3" "9090" "c3" "c3")
679
+ no_return = [{"routine": 4, "reason": "synthetic: declared to end the program"}]
680
+ declared = [{"site": 7, "exhaustive": True, "targets": [0x0D], "evidence": "synthetic: bx holds 000D"}]
681
+ r = run_report(data, config(data, targets=[3], controls=[0], noReturn=no_return, indirectCalls=declared,
682
+ stops=[{"routine": 0x0C, "reason": CUT}]), "reach")
683
+ self.assertEqual((r["noReturn"][0]["returnSites"], r["noReturn"][0]["contradicted"]), ([9], True))
684
+ self.assertFalse(r["negativeUsable"])
685
+
686
+ def test_the_check_enters_a_declared_routine_the_query_reaches_only_past_a_stop(self):
687
+ # R1 at 0004 calls the stop routine S at 000D, then R2 at 000E, which returns. Only the check
688
+ # reaches R2, and its return contradicts R2's declaration.
689
+ data = bytes.fromhex("e80100" "c3" "e80600" "e80400" "ebfe" "90" "c3" "c3")
690
+ no_return = [{"routine": 4, "reason": "synthetic: declared to end the program"},
691
+ {"routine": 0x0E, "reason": "synthetic: declared to end the program"}]
692
+ r = run_report(data, config(data, targets=[3], controls=[0], noReturn=no_return,
693
+ stops=[{"routine": 0x0D, "reason": CUT}]), "reach")
694
+ self.assertEqual([(row["reached"], row["read"], row["returnSites"], row["contradicted"]) for row in r["noReturn"]],
695
+ [(True, True, [], False), (False, True, [0x0E], True)])
696
+ self.assertFalse(r["negativeUsable"])
697
+
698
+ def test_the_no_return_checks_callees_do_not_spend_the_instruction_limit(self):
699
+ # R's callee at 000B holds nine instructions; a check that entered it would trip a limit of
700
+ # six, where R's own path holds two.
701
+ data = bytes.fromhex("e80100" "c3" "e80400" "ebfe" "9090" + "90" * 8 + "c3")
702
+ no_return = [{"routine": 4, "reason": "synthetic: declared to end the program"}]
703
+ r = run_report(data, config(data, targets=[3], controls=[0], noReturn=no_return,
704
+ stops=[{"routine": 0x0B, "reason": CUT}], instructionLimit=6), "reach")
705
+ self.assertFalse(r["instructionLimitReached"])
706
+ self.assertFalse(r["noReturn"][0]["contradicted"])
707
+ self.assertTrue(r["negativeUsable"])
708
+
634
709
  def test_rejected_stops(self):
635
710
  site = {"site": 3, "reason": CUT}
636
711
  for extra, message in [({"stops": {}}, "stops must be a list"),
@@ -0,0 +1,163 @@
1
+ """Persistent scans use synthetic bytes only and never qualify raw candidates."""
2
+ import copy
3
+ from contextlib import closing
4
+ import os
5
+ from pathlib import Path
6
+ import sqlite3
7
+ import sys
8
+ import tempfile
9
+ import unittest
10
+ from unittest.mock import patch
11
+
12
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
13
+ from scientific_method_engine.x86.reports import run_report
14
+ from scientific_method_engine.x86.image import Image
15
+
16
+
17
+ class ScanIndexTests(unittest.TestCase):
18
+ def setUp(self):
19
+ self.folder = tempfile.TemporaryDirectory()
20
+ self.addCleanup(self.folder.cleanup)
21
+ # MOV AX,1234; MOV AX,[BX+44]; CALL 000C; CALL AX; RET.
22
+ self.data = bytes.fromhex("b834128b4744e80300ffd0c3c3")
23
+ self.config = {"sourceKind": "synthetic-raw", "regions": [
24
+ {"name": "code", "start": 0, "end": len(self.data), "ip": 0,
25
+ "segment": 4096, "entries": [0], "evidence": "synthetic"}],
26
+ "scanIndex": {"directory": self.folder.name},
27
+ "query": {"kind": "immediate", "value": 0x1234, "controls": [0]}}
28
+
29
+ def report(self, config=None, data=None):
30
+ return run_report(self.data if data is None else data, config or self.config, "scan-index")
31
+
32
+ def test_reuses_corpus_for_different_operands_without_decoding(self):
33
+ first = self.report()
34
+ self.assertFalse(first["cacheHit"])
35
+ self.assertEqual(first["rawCandidates"], [{"site": 0, "size": 3}])
36
+ config = copy.deepcopy(self.config)
37
+ config["query"] = {"kind": "displacement", "value": 0x44, "controls": [3]}
38
+ with patch.object(Image, "decode", side_effect=AssertionError("cache hit decoded again")):
39
+ second = self.report(config)
40
+ self.assertTrue(second["cacheHit"])
41
+ self.assertEqual(first["cacheKey"], second["cacheKey"])
42
+ self.assertEqual(second["rawCandidates"], [{"site": 3, "size": 3}])
43
+
44
+ def test_calls_controls_and_result_limit(self):
45
+ config = copy.deepcopy(self.config)
46
+ config["query"] = {"kind": "call", "value": 12, "controls": [6]}
47
+ self.assertIn({"site": 6, "size": 3}, self.report(config)["rawCandidates"])
48
+ config["query"]["controls"] = [0]
49
+ with self.assertRaisesRegex(ValueError, "controls missed"):
50
+ self.report(config)
51
+ repeated = bytes.fromhex("b83412b83412c3")
52
+ config["regions"][0]["end"] = len(repeated)
53
+ config["query"] = {"kind": "immediate", "value": 0x1234, "limit": 1}
54
+ report = self.report(config, repeated)
55
+ self.assertTrue(report["resultLimitReached"])
56
+ self.assertEqual(report["totalCandidates"], 2)
57
+ self.assertEqual(len(report["rawCandidates"]), 1)
58
+
59
+ def test_reports_where_the_scan_could_not_see(self):
60
+ report = self.report()
61
+ self.assertEqual(report["unresolvedCalls"], [{"site": 9, "size": 2}])
62
+ self.assertFalse(report["unresolvedCallsLimitReached"])
63
+ self.assertEqual(report["undecodedRanges"], [])
64
+ # RET, then MOV AX,imm16 and XOR AL,imm8 cut off by each region's end.
65
+ data = bytes.fromhex("c3b834c3b834")
66
+ config = copy.deepcopy(self.config)
67
+ config["regions"] = [
68
+ {"name": "a", "start": 0, "end": 3, "ip": 0, "segment": 4096, "entries": [0], "evidence": "synthetic"},
69
+ {"name": "b", "start": 3, "end": 6, "ip": 0, "segment": 8192, "entries": [3], "evidence": "synthetic"}]
70
+ config["query"] = {"kind": "immediate", "value": 0x1234}
71
+ first = self.report(config, data)
72
+ self.assertEqual(first["undecodedRanges"], [{"start": 1, "end": 3}, {"start": 4, "end": 6}])
73
+ self.assertEqual(first["scan"]["undecodedOffsets"], 4)
74
+ config["query"]["limit"] = 1
75
+ with patch.object(Image, "decode", side_effect=AssertionError("cache hit decoded again")):
76
+ capped = self.report(config, data)
77
+ self.assertTrue(capped["cacheHit"])
78
+ self.assertEqual(capped["undecodedRanges"], [{"start": 1, "end": 3}])
79
+ self.assertTrue(capped["undecodedRangesLimitReached"])
80
+
81
+ def test_changed_source_mapping_entries_and_decoder_identity_invalidate(self):
82
+ first = self.report()["cacheKey"]
83
+ self.assertNotEqual(first, self.report(data=self.data[:-1] + b"\x90")["cacheKey"])
84
+ for field, value in (("ip", 32), ("entries", [3]), ("evidence", "changed mapping")):
85
+ config = copy.deepcopy(self.config)
86
+ config["regions"][0][field] = value
87
+ self.assertNotEqual(first, self.report(config)["cacheKey"])
88
+ with patch("scientific_method_engine.x86.scan_index.CAPSTONE_VERSION", "different"):
89
+ self.assertNotEqual(first, self.report()["cacheKey"])
90
+
91
+ def test_incomplete_scan_writes_nothing_and_invalid_options_fail(self):
92
+ config = copy.deepcopy(self.config)
93
+ config["scanLimit"] = 2
94
+ with self.assertRaisesRegex(ValueError, "scanLimit reached"):
95
+ self.report(config)
96
+ self.assertEqual(list(Path(self.folder.name).iterdir()), [])
97
+ config = copy.deepcopy(self.config)
98
+ config["scanIndex"]["directory"] = "relative"
99
+ with self.assertRaisesRegex(ValueError, "absolute"):
100
+ self.report(config)
101
+ for query in ({"kind": "unknown", "value": 0}, {"kind": "call", "value": True},
102
+ {"kind": "call", "value": 0, "limit": 0},
103
+ {"kind": "call", "value": 0, "limit": 10001},
104
+ {"kind": "call", "value": 0, "controls": [0] * 129},
105
+ {"kind": "call", "value": 0, "controls": [999]}):
106
+ config = copy.deepcopy(self.config)
107
+ config["query"] = query
108
+ with self.assertRaises(ValueError):
109
+ self.report(config)
110
+
111
+ def test_corrupt_or_mismatched_cache_is_refused(self):
112
+ report = self.report()
113
+ path = Path(self.folder.name) / (report["cacheKey"] + ".sqlite")
114
+ with closing(sqlite3.connect(path)) as db, db:
115
+ db.execute("UPDATE metadata SET value='{}'")
116
+ with self.assertRaisesRegex(ValueError, "identity differs"):
117
+ self.report()
118
+ path.write_bytes(b"not a database")
119
+ with self.assertRaisesRegex(ValueError, "cache failed"):
120
+ self.report()
121
+
122
+ def test_implicit_operands_branch_targets_and_prefixed_calls_are_not_candidates(self):
123
+ # MOV AX,[BX]; SHL AX,1; JE 0006; CALL with an operand-size override to 000C; RET.
124
+ data = bytes.fromhex("8b07d1e0740066e800000000c3")
125
+ config = copy.deepcopy(self.config)
126
+ config["regions"][0]["end"] = len(data)
127
+ for kind, value in (("displacement", 0), ("immediate", 1), ("immediate", 6), ("call", 12)):
128
+ config["query"] = {"kind": kind, "value": value}
129
+ self.assertEqual(self.report(config, data)["totalCandidates"], 0, (kind, value))
130
+ # The unprefixed CALL decoded inside the prefixed one still resolves.
131
+ config["query"] = {"kind": "call", "value": 10, "controls": [7]}
132
+ self.assertEqual(self.report(config, data)["rawCandidates"], [{"site": 7, "size": 3}])
133
+
134
+ def test_flat32_has_separate_mapping_and_candidates(self):
135
+ data = bytes.fromhex("b878563412c3")
136
+ config = copy.deepcopy(self.config)
137
+ config.update(bits=32, addressModel="flat32")
138
+ config["regions"][0].update(end=len(data), segment=0, ip=0x1000)
139
+ config["query"] = {"kind": "immediate", "value": 0x12345678, "controls": [0]}
140
+ self.assertEqual(self.report(config, data)["rawCandidates"], [{"site": 0, "size": 5}])
141
+
142
+ def test_concurrent_publication_of_the_same_corpus_is_kept(self):
143
+ # Windows refuses the replace while another run has the published corpus open.
144
+ replace = os.replace
145
+
146
+ def published_by_another_run(source, destination):
147
+ replace(source, destination)
148
+ raise PermissionError("in use")
149
+ with patch("scientific_method_engine.x86.scan_index.os.replace", side_effect=published_by_another_run):
150
+ report = self.report()
151
+ self.assertEqual(report["rawCandidates"], [{"site": 0, "size": 3}])
152
+ self.assertEqual(list(Path(self.folder.name).glob('*.partial')), [])
153
+
154
+ def test_disk_budget_failure_never_publishes_a_partial_index(self):
155
+ with patch("scientific_method_engine.x86.scan_index.MAX_INDEX_BYTES", 4096):
156
+ with self.assertRaisesRegex(ValueError, "build failed and published nothing"):
157
+ self.report()
158
+ self.assertEqual(list(Path(self.folder.name).glob('*.sqlite')), [])
159
+ self.assertEqual(list(Path(self.folder.name).glob('*.partial')), [])
160
+
161
+
162
+ if __name__ == "__main__":
163
+ unittest.main()