scientific-method-engine 19.3.0__tar.gz → 19.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/PKG-INFO +1 -1
  2. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/pyproject.toml +1 -1
  3. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/cli.py +4 -0
  4. scientific_method_engine-19.4.0/src/scientific_method_engine/x86/derived_counts.py +171 -0
  5. scientific_method_engine-19.4.0/src/scientific_method_engine/x86/dispatch.py +336 -0
  6. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/x86/inventory.py +2 -1
  7. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/x86/reach.py +4 -2
  8. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/x86/reports.py +17 -8
  9. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/x86/table_bounds.py +25 -7
  10. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/x86/trace.py +28 -11
  11. scientific_method_engine-19.4.0/tests/test_derived_counts.py +401 -0
  12. scientific_method_engine-19.3.0/src/scientific_method_engine/x86/dispatch.py +0 -168
  13. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/.gitignore +0 -0
  14. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/LICENSE +0 -0
  15. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/README.md +0 -0
  16. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/__init__.py +0 -0
  17. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/__main__.py +0 -0
  18. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  19. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  20. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  21. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  22. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  23. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  24. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  25. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  26. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  27. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  28. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  29. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  30. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  31. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  32. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  33. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  34. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  35. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  36. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  37. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  38. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  39. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  40. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  41. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  42. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  43. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  44. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
  45. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  46. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  47. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  48. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  49. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
  50. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
  51. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
  52. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
  53. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  54. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
  55. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  56. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/x86/image.py +0 -0
  57. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/x86/loops.py +0 -0
  58. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/x86/machine.py +0 -0
  59. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
  60. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/x86/pcode.py +0 -0
  61. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
  62. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/x86/pe.py +0 -0
  63. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/x86/ranges.py +0 -0
  64. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/x86/relational.py +0 -0
  65. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  66. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/src/scientific_method_engine/x86/values.py +0 -0
  67. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/oracle.py +0 -0
  68. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_config_encoding.py +0 -0
  69. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_continuation_budget.py +0 -0
  70. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_dispatch.py +0 -0
  71. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_effect_order.py +0 -0
  72. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_enter_frames.py +0 -0
  73. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_entry_frame.py +0 -0
  74. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_indirect_far_transfers.py +0 -0
  75. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_interrupt_models.py +0 -0
  76. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_inventory_check.py +0 -0
  77. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_join_terms.py +0 -0
  78. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_loops.py +0 -0
  79. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_memory_scopes.py +0 -0
  80. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_nested_frame_request.py +0 -0
  81. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_oracle.py +0 -0
  82. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_pe.py +0 -0
  83. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_ranges.py +0 -0
  84. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_reach.py +0 -0
  85. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_relational_controls.py +0 -0
  86. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_return_checks.py +0 -0
  87. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_successor_rule.py +0 -0
  88. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_table_bounds.py +0 -0
  89. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_table_continuations.py +0 -0
  90. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_term_limit.py +0 -0
  91. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_upstream_gaps.py +0 -0
  92. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_volatile_memory.py +0 -0
  93. {scientific_method_engine-19.3.0 → scientific_method_engine-19.4.0}/tests/test_x86.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 19.3.0
3
+ Version: 19.4.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "19.3.0"
8
+ version = "19.4.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -33,6 +33,10 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
33
33
  "the noReturn check reads past the stops, and returnCheckGaps lists what it could not read there;\n"
34
34
  "indirectCalls declares the targets of a computed call as a table read from the build or a list,\n"
35
35
  "and the walk calls each of them; one not declared exhaustive stays unresolved;\n"
36
+ "an exhaustive indirectJumps or indirectCalls table can leave out count and give bound {routine}:\n"
37
+ "the engine derives its rows from a value-range analysis of that routine and lists the derivation,\n"
38
+ "a count it cannot prove leaves the site unresolved with its reasons, a declared count larger than\n"
39
+ "the proven one is refused, and a walk edge that enters the routine other than at its start is a gap;\n"
36
40
  "controls are call sites the walk must reach and resolve, instructionControls sites it must decode,\n"
37
41
  "and a failed control fails the report saying whether it was reached;\n"
38
42
  "decodedRanges true lists every instruction the walk decoded as half-open ranges with their routine.\n"
@@ -0,0 +1,171 @@
1
+ """The rows a declared dispatch table can be read at, derived from the code (ADR 0034).
2
+
3
+ A declaration's ``table.bound`` names the routine whose code bounds the dispatch's index. ``derive``
4
+ runs ``table_bounds.analyse`` from that routine's start to the dispatch site and maps the proven
5
+ range of the offset the dispatch reads its row at onto the table's rows. The table's start, stride
6
+ and field offset stay declared, and so does the segment offset of its first row, which the code
7
+ segment's mapping of the dispatch site gives for a table read through CS.
8
+
9
+ The analysis reads only paths that enter the routine at its start. ``entry_gaps`` checks that
10
+ claim against the edges a command's own walk read, and each command reports a breach as a gap at
11
+ the dispatch site.
12
+ """
13
+ from .dispatch import ENTERED
14
+ from .image import integer
15
+ from .table_bounds import PASSES, analyse
16
+
17
+
18
+ STRIDE = "an index that does not step by the table stride"
19
+ BEFORE = "an offset before the table's first row"
20
+ WIDTH = "a dispatch that reads another width than the table's rows hold"
21
+ NO_OFFSET = "a table read through a segment other than CS whose offset is not declared"
22
+ MAPPING = "a table offset the dispatch site's code segment does not place"
23
+ SOURCE = "rows that leave the source bytes"
24
+ TARGET = "a row whose target is not admitted"
25
+ UNSETTLED = "rows that did not settle"
26
+ DEPENDS = "a dispatch the analysis continued through whose rows are not proven"
27
+ # Rounds of deriving every declared jump table again with the rows the earlier rounds gave.
28
+ ROUNDS = 64
29
+
30
+
31
+ def bound_input(value, label, image):
32
+ """The ``bound`` object of a declared table: the routine to derive the count over and the analysis limits."""
33
+ if not isinstance(value, dict) or "routine" not in value or set(value) - {"routine", "tableOffset", "passes",
34
+ "instructionLimit"}:
35
+ raise ValueError(f"{label} bound needs routine, and takes only tableOffset, passes and instructionLimit")
36
+ routine = integer(value["routine"], 0, len(image.data) - 1, f"{label} bound routine")
37
+ if image.region(routine) is None:
38
+ raise ValueError(f"{label} bound routine {routine} is outside declared code")
39
+ offset = value.get("tableOffset")
40
+ if offset is not None:
41
+ offset = integer(offset, 0, 0xFFFF, f"{label} bound tableOffset")
42
+ return {"routine": routine, "tableOffset": offset,
43
+ "passes": integer(value.get("passes", PASSES), 1, 64, f"{label} bound passes"),
44
+ "instructionLimit": integer(value.get("instructionLimit", 10000), 1, 10000,
45
+ f"{label} bound instructionLimit")}
46
+
47
+
48
+ def code_offset(image, site, at):
49
+ """The offset in the code segment of the dispatch at ``site`` that places the source byte ``at``, or None.
50
+
51
+ A byte inside a declared region is placed through that region's mapping; one outside every region
52
+ is read as an extension of the site's region, as a traced dispatch reads its table words.
53
+ """
54
+ region, home = image.region(at) or image.region(site), image.region(site)
55
+ offset = region["segment"] * 16 + region["ip"] + at - region["start"] - home["segment"] * 16
56
+ return offset if 0 <= offset <= 0xFFFF else None
57
+
58
+
59
+ def derive(image, site, layout, width, bound, through, make_row, continuing=None):
60
+ """The rows the dispatch at ``site`` can read, from the analysis over ``bound``'s routine.
61
+
62
+ ``layout`` is the table's ``(start, count, stride, fieldOffset)`` with ``count`` None when it is
63
+ not declared, ``width`` the bytes a row's target takes, ``through`` the targets a dispatch jump
64
+ continues at during the analysis, ``continuing`` the targets other declared jumps continue at
65
+ in place of their declarations (``table_bounds.analyse``), and ``make_row(operand, index)``
66
+ gives the row whose target field is at ``operand``, as a row dict or a reason string.
67
+ Returns the derivation record, the rows it admits (None unless proven) and the successors the
68
+ analysis read from each site.
69
+ """
70
+ start, _, stride, field = layout
71
+ result, graph, offset = analyse(image, bound["routine"], site, through, bound["passes"],
72
+ bound["instructionLimit"], continuing)
73
+ table = result["table"] or {}
74
+ record = {"routine": bound["routine"], "proven": False, "segment": table.get("segment"),
75
+ "offset": table.get("offset"), "tableOffset": bound["tableOffset"], "rowIndexes": None, "count": None,
76
+ "bounds": result["bounds"], "reasons": list(result["reasons"]), "unread": result["unread"],
77
+ "widened": result["widened"], "instructions": result["instructions"],
78
+ "instructionLimitReached": result["instructionLimitReached"]}
79
+ rows = None
80
+ if result["proven"]:
81
+ rows = _rows(image, site, start, stride, field, width, bound, table, offset, record, make_row)
82
+ record["proven"] = not record["reasons"]
83
+ if not record["proven"]:
84
+ rows = None
85
+ record["assumptions"] = _assumptions(record, bound)
86
+ return record, rows, graph
87
+
88
+
89
+ def _rows(image, site, start, stride, field, width, bound, table, offset, record, make_row):
90
+ def refuse(reason, **detail):
91
+ record["reasons"].append({"reason": reason, "site": site, **detail})
92
+ if table["width"] != width:
93
+ refuse(WIDTH, read=table["width"], width=width)
94
+ return None
95
+ first_field = start + field
96
+ placed = code_offset(image, site, first_field) if table["segment"] == "CS" else None
97
+ if bound["tableOffset"] is None:
98
+ if table["segment"] != "CS":
99
+ refuse(NO_OFFSET)
100
+ return None
101
+ if placed is None:
102
+ refuse(MAPPING)
103
+ return None
104
+ record["tableOffset"] = placed
105
+ elif table["segment"] == "CS" and placed != bound["tableOffset"]:
106
+ refuse(MAPPING, placed=placed)
107
+ return None
108
+ base = record["tableOffset"]
109
+ if offset.lo < base:
110
+ refuse(BEFORE)
111
+ return None
112
+ if (offset.lo - base) % stride or offset.stride % stride:
113
+ refuse(STRIDE)
114
+ return None
115
+ first, last = (offset.lo - base) // stride, (offset.hi - base) // stride
116
+ step = offset.stride // stride or 1
117
+ if first_field + last * stride + width > len(image.data):
118
+ refuse(SOURCE)
119
+ return None
120
+ rows = []
121
+ for index in range(first, last + 1, step):
122
+ row = make_row(start + index * stride + field, index)
123
+ if isinstance(row, str):
124
+ refuse(TARGET, row=index, detail=row)
125
+ return None
126
+ rows.append(row)
127
+ record["rowIndexes"] = {"first": first, "last": last, "step": step}
128
+ record["count"] = last + 1
129
+ return rows
130
+
131
+
132
+ def _assumptions(record, bound):
133
+ found = [f"the routine at {bound['routine']} is entered only at its start; each command checks this against "
134
+ "the edges its own walk reads",
135
+ "each call and interrupt the analysis steps over returns to its next instruction"]
136
+ offset = record["tableOffset"]
137
+ if offset is None:
138
+ return found
139
+ if bound["tableOffset"] is None:
140
+ return found + [f"CS holds the segment of the dispatch site's region, so the dispatch reads the first row's "
141
+ f"target field at offset {offset}"]
142
+ return found + [f"the dispatch reads the first row's target field at the declared offset {offset} of "
143
+ f"{record['segment'] or 'the segment it reads through'}"]
144
+
145
+
146
+ def entry_gaps(image, entries, successors, indirect_calls=None):
147
+ """The gaps of a walk that entered a routine a read count was derived over other than at its start.
148
+
149
+ ``entries`` are the walk's starts and ``successors`` the sites the walk read from each site. The
150
+ analyses are the declared jumps' (``image.derived_reads``) and those of the ``indirect_calls``
151
+ sites the walk takes (``image.derived_call_reads``), each as ``(routine, successors read)``. A
152
+ derived declaration counts when the walk read its dispatch site. An edge into a site the
153
+ analysis read, other than the routine's start, breaches the derivation unless the analysis read
154
+ that same edge.
155
+ """
156
+ reads = dict(image.derived_reads)
157
+ calls = getattr(image, "derived_call_reads", {})
158
+ reads |= {at: calls[at] for at in (indirect_calls or {}) if at in calls}
159
+ gaps = []
160
+ for site, (routine, graph) in sorted(reads.items()):
161
+ if site not in successors:
162
+ continue
163
+ breaches = sorted({(None, at) for at in entries if at in graph and at != routine}
164
+ | {(at, s) for at, found in successors.items() for s in found
165
+ if s in graph and s != routine and not (at in graph and s in graph[at])},
166
+ key=lambda e: (e[1], -1 if e[0] is None else e[0]))
167
+ if breaches:
168
+ source, target = breaches[0]
169
+ gaps.append({"site": site, "reason": ENTERED, "routine": routine, "entry": target,
170
+ **({"from": source} if source is not None else {})})
171
+ return gaps
@@ -0,0 +1,336 @@
1
+ """Explicit, source-derived indirect jump tables and call targets for CFG discovery, and explicitly conditional path continuations."""
2
+ from capstone.x86 import X86_OP_IMM, X86_OP_MEM
3
+ from .image import integer
4
+
5
+ CALL_NOT_EXHAUSTIVE = "indirect call targets are not declared exhaustive"
6
+ JUMP_NOT_EXHAUSTIVE = "indirect jump table is not declared exhaustive"
7
+ # The reason a dispatch whose count the analysis did not prove stays unresolved.
8
+ JUMP_NOT_DERIVED = "indirect jump table count is not proven by the code"
9
+ CALL_NOT_DERIVED = "indirect call table count is not proven by the code"
10
+ # The gap a walk records when one of its edges enters an analysed routine other than at its start.
11
+ ENTERED = "a path enters the routine a table count was derived over other than at its start"
12
+
13
+
14
+ def _word(image, at):
15
+ return int.from_bytes(image.data[at:at + 2], "little")
16
+
17
+
18
+ def _capital(label):
19
+ return label[0].upper() + label[1:]
20
+
21
+
22
+ def _layout(image, table, width, label, field_label):
23
+ """A declared ``table``'s ``(start, count, stride, fieldOffset)``, with ``count`` None when it is not declared."""
24
+ start = integer(table.get("start"), 0, len(image.data) - 1, f"{label} start")
25
+ count = integer(table["count"], 1, 256, f"{label} count") if "count" in table else None
26
+ stride = integer(table.get("stride"), width, 65536, f"{label} stride")
27
+ field = integer(table.get("fieldOffset", 0), 0, stride - width, field_label)
28
+ if count is not None and start + (count - 1) * stride + field + width > len(image.data):
29
+ raise ValueError(f"{_capital(label)} leaves source bounds")
30
+ return start, count, stride, field
31
+
32
+
33
+ def _bound(image, table, layout, exhaustive, label):
34
+ """The table's parsed ``bound``, or None. A table without ``count`` needs one, and an exhaustive declaration."""
35
+ from .derived_counts import bound_input
36
+ bound = bound_input(table["bound"], label, image) if "bound" in table else None
37
+ if layout[1] is None:
38
+ if bound is None:
39
+ raise ValueError(f"{_capital(label)} needs count, or bound to derive the count from the code")
40
+ if not exhaustive:
41
+ raise ValueError(f"{_capital(label)} without count needs exhaustive true: only an exhaustive table takes "
42
+ "its count from the code")
43
+ return bound
44
+
45
+
46
+ def _rows(layout, indexes, make_row):
47
+ start, _, stride, field = layout
48
+ rows = []
49
+ for index in indexes:
50
+ row = make_row(start + index * stride + field, index)
51
+ if isinstance(row, str):
52
+ raise ValueError(row)
53
+ rows.append(row)
54
+ return rows
55
+
56
+
57
+ def _check_count(site, layout, derivation, label):
58
+ """Compare a declared count with a proven derivation: a larger one is refused, a smaller one followed."""
59
+ count = layout[1]
60
+ derivation["declaredCount"] = count
61
+ derivation["rowsLeftOut"] = []
62
+ if not derivation["proven"]:
63
+ return
64
+ if count > derivation["count"]:
65
+ raise ValueError(f"{_capital(label)} at {site} declares count {count}, but the code bounds the rows its "
66
+ f"dispatch can read to {derivation['count']}; the rows past them cannot be read")
67
+ found = derivation["rowIndexes"]
68
+ derivation["rowsLeftOut"] = [i for i in range(found["first"], found["last"] + 1, found["step"]) if i >= count]
69
+
70
+
71
+ def _jump_row(image, site):
72
+ def make_row(operand, index):
73
+ raw = _word(image, operand)
74
+ target = image.near_target(site, raw)
75
+ if target is None:
76
+ return "Indirect table target leaves declared code mappings"
77
+ return {"index": index, "operandSite": operand, "rawOffset": raw, "target": target}
78
+ return make_row
79
+
80
+
81
+ def read_indirect_jumps(image):
82
+ """The declared indirect jump tables by site, with their rows; also sets ``image.indirect_jumps`` to them.
83
+
84
+ A table without ``count`` takes its rows from the code (``derived_counts.derive``), and one with
85
+ both ``count`` and ``bound`` is checked against the code. A jump's rows feed its own analysis and
86
+ every analysis whose routine holds the jump, so the tables without ``count`` are derived in
87
+ rounds (``_derive_jumps``), and the tables with both are checked once those rows are final. The
88
+ successors each proven analysis of a table without ``count`` read are kept in
89
+ ``image.derived_reads`` for ``entry_gaps``.
90
+ """
91
+ from .derived_counts import derive
92
+ declarations = image.config.get("indirectJumps", [])
93
+ if not isinstance(declarations, list) or len(declarations) > 256:
94
+ raise ValueError("indirectJumps must contain at most 256 declarations")
95
+ if declarations and image.flat:
96
+ raise ValueError("indirectJumps currently requires segmented16")
97
+ result, pending = {}, {}
98
+ for claim in declarations:
99
+ if not isinstance(claim, dict):
100
+ raise ValueError("Each indirect jump needs an object")
101
+ site = integer(claim.get("site"), 0, len(image.data) - 1, "indirect jump site")
102
+ if site in result:
103
+ raise ValueError("Duplicate indirect jump site")
104
+ if not isinstance(claim.get("evidence"), str) or not claim["evidence"].strip():
105
+ raise ValueError("An indirect jump needs consumer/mapping evidence")
106
+ if type(claim.get("exhaustive")) is not bool:
107
+ raise ValueError("An indirect jump needs explicit exhaustive true or false")
108
+ ins = image.decode(site)
109
+ if (ins is None or ins.mnemonic != "jmp" or len(ins.operands) != 1
110
+ or ins.operands[0].type == X86_OP_IMM or ins.operands[0].size != 2
111
+ or 0x66 in ins.prefix or 0x67 in ins.prefix):
112
+ raise ValueError("indirectJumps site must decode as an unprefixed computed near word jump")
113
+ table = claim.get("table")
114
+ if not isinstance(table, dict) or not isinstance(table.get("evidence"), str) or not table["evidence"].strip():
115
+ raise ValueError("Indirect table needs layout/count evidence")
116
+ if type(table.get("width", 2)) is not int or table.get("width", 2) != 2:
117
+ raise ValueError("Indirect table targets must be word width 2")
118
+ layout = _layout(image, table, 2, "indirect table", "indirect target field offset")
119
+ bound = _bound(image, table, layout, claim["exhaustive"], "indirect table")
120
+ rows = _rows(layout, range(layout[1] or 0), _jump_row(image, site))
121
+ result[site] = {"site": site, "evidence": claim["evidence"], "table": table,
122
+ "exhaustive": claim["exhaustive"], "rows": rows,
123
+ "interpretation": "source words under supplied consumer, mapping and exhaustiveness evidence; "
124
+ "not executed dispatch or an overlap-boundary proof"}
125
+ if bound is not None:
126
+ pending[site] = (layout, bound)
127
+ result[site]["derivation"] = None
128
+ image.indirect_jumps, image.derived_reads = result, {}
129
+ _derive_jumps(image, result, {site: entry for site, entry in pending.items() if entry[0][1] is None})
130
+ # A declared count's rows feed no derivation, so it is checked once against the final rows of the others.
131
+ for site, (layout, bound) in pending.items():
132
+ if layout[1] is not None:
133
+ declaration = result[site]
134
+ declaration["derivation"] = derive(image, site, layout, 2, bound,
135
+ [row["target"] for row in declaration["rows"]],
136
+ _jump_row(image, site))[0]
137
+ _check_count(site, layout, declaration["derivation"], "indirect table")
138
+ return result
139
+
140
+
141
+ def _derive_jumps(image, result, derived):
142
+ """Give each jump table in ``derived`` (site to ``(layout, bound)``) its rows from the code.
143
+
144
+ Each round derives every such table with each of them continuing at every row any earlier round
145
+ gave it, starting from none, so the rows only grow. In a round that adds none, every analysis
146
+ ran with the final rows. A table whose rows still grew in the last round is not proven, nor is
147
+ one whose analysis continued through a table that is not proven.
148
+ """
149
+ from .derived_counts import DEPENDS, ROUNDS, UNSETTLED, derive
150
+ given = {site: {} for site in derived}
151
+ records, graphs, grew = {}, {}, set()
152
+ for _ in range(ROUNDS if derived else 0):
153
+ grew = set()
154
+ for site, (layout, bound) in derived.items():
155
+ targets = {at: [row["target"] for row in rows.values()] for at, rows in given.items()}
156
+ through = targets.pop(site)
157
+ record, found, graph = derive(image, site, layout, 2, bound, through, _jump_row(image, site), targets)
158
+ records[site], graphs[site] = (record, found or []), graph
159
+ if any(row["index"] not in given[site] for row in found or []):
160
+ grew.add(site)
161
+ given[site] |= {row["index"]: row for row in found}
162
+ if not grew:
163
+ break
164
+ unproven = grew | {site for site, (record, _) in records.items() if not record["proven"]}
165
+ while True:
166
+ depending = {site for site in derived if site not in unproven and unproven & graphs[site].keys()}
167
+ if not depending:
168
+ break
169
+ unproven |= depending
170
+ for site, (layout, bound) in derived.items():
171
+ record, found = records[site]
172
+ declaration = result[site]
173
+ declaration["derivation"] = record
174
+ if site not in unproven:
175
+ declaration["rows"] = found
176
+ image.derived_reads[site] = (bound["routine"], graphs[site])
177
+ continue
178
+ declaration["rows"] = []
179
+ if site in grew:
180
+ record["reasons"].append({"reason": UNSETTLED, "site": site, "rounds": ROUNDS})
181
+ if record["proven"]:
182
+ record["reasons"] += [{"reason": DEPENDS, "site": at} for at in sorted(unproven & graphs[site].keys())
183
+ if at != site]
184
+ record |= {"proven": False, "rowIndexes": None, "count": None}
185
+
186
+
187
+ def _call_row(image, site, far):
188
+ def make_row(operand, index):
189
+ row = {"index": index, "operandSite": operand, "rawOffset": _word(image, operand)}
190
+ if far:
191
+ # The segment word is relocated at load time, and the pointer is admitted as a traced far
192
+ # call through memory admits the pointer it reads (Image.far_pointer_target).
193
+ fixup = image.fixups.get(operand + 2)
194
+ if fixup is None:
195
+ return (f"Indirect call table row {index} has no declared relocation for its segment word at "
196
+ f"{operand + 2}")
197
+ target, admission = image.far_pointer_target(fixup["segment"], row["rawOffset"])
198
+ if target is None:
199
+ return f"Indirect call table row {index} target is not admitted: {admission['reason']}"
200
+ row |= {"rawSegment": _word(image, operand + 2), "resolvedSegment": fixup["segment"]}
201
+ # A pointer at an FBOV trampoline continues at its overlay entry, which the row names.
202
+ if "trampoline" in admission:
203
+ row["trampoline"] = admission["trampoline"]
204
+ else:
205
+ target = image.near_target(site, row["rawOffset"])
206
+ if target is None or image.region(target) is None:
207
+ return f"Indirect call table row {index} target leaves declared code mappings"
208
+ return row | {"target": target}
209
+ return make_row
210
+
211
+
212
+ def indirect_call_declarations(value, image):
213
+ """The declared targets of computed calls, by call site, as ``reach`` and ``inventory-check`` take them.
214
+
215
+ Each declaration names a ``site`` that decodes as an unprefixed segmented16 computed call (a near
216
+ call through a word register or memory operand, or a far call through memory), its ``evidence``,
217
+ an explicit boolean ``exhaustive``, and exactly one of ``table`` and ``targets``. A table's rows are
218
+ read from the source bytes as ``indirectJumps`` reads them: words placed through the call site's
219
+ region mapping for a near call, and for a far call ``offset, segment`` word pairs whose segment
220
+ word has a declared relocation, admitted as ``Image.far_pointer_target`` admits the pointer a
221
+ traced far call reads; a far row whose pointer names an FBOV trampoline carries its
222
+ ``trampoline``. A table without ``count`` takes its rows from the code, as ``indirectJumps`` does,
223
+ and one with both ``count`` and ``bound`` is checked against the code. ``targets`` gives file
224
+ offsets in declared code, which rest on the evidence alone; a near call's targets must be bytes
225
+ some IP in the call site's segment places. Each declaration is returned with the call's
226
+ ``instruction`` text, a table's ``rows`` and, with a ``bound``, its ``derivation``, and its
227
+ distinct ``targets`` in row order. The successors each proven analysis of a table without
228
+ ``count`` read are kept in ``image.derived_call_reads`` for ``entry_gaps``.
229
+ """
230
+ from .derived_counts import derive
231
+ if not isinstance(value, list) or len(value) > 256:
232
+ raise ValueError("indirectCalls must be a list of at most 256 declarations")
233
+ if value and image.flat:
234
+ raise ValueError("indirectCalls currently requires segmented16")
235
+ result = {}
236
+ image.derived_call_reads = {}
237
+ for claim in value:
238
+ keys = set(claim) if isinstance(claim, dict) else None
239
+ if keys not in ({"site", "evidence", "exhaustive", "table"}, {"site", "evidence", "exhaustive", "targets"}):
240
+ raise ValueError("Each indirect call needs site, evidence, exhaustive and exactly one of table and targets")
241
+ site = integer(claim["site"], 0, len(image.data) - 1, "indirect call site")
242
+ if site in result:
243
+ raise ValueError("Duplicate indirect call site")
244
+ if not isinstance(claim["evidence"], str) or not claim["evidence"].strip():
245
+ raise ValueError("An indirect call needs evidence that connects the call's operand to its targets")
246
+ if type(claim["exhaustive"]) is not bool:
247
+ raise ValueError("An indirect call needs explicit exhaustive true or false")
248
+ ins = image.decode(site)
249
+ far = ins is not None and ins.mnemonic == "lcall"
250
+ if (ins is None or ins.mnemonic not in ("call", "lcall") or len(ins.operands) != 1
251
+ or ins.operands[0].type == X86_OP_IMM or (far and ins.operands[0].type != X86_OP_MEM)
252
+ or (not far and ins.operands[0].size != 2) or 0x66 in ins.prefix or 0x67 in ins.prefix):
253
+ raise ValueError(f"indirect call site {site} must decode as an unprefixed computed near word call "
254
+ "or a far call through memory")
255
+ declaration = {"site": site, "instruction": (ins.mnemonic + " " + ins.op_str).strip(),
256
+ "evidence": claim["evidence"], "exhaustive": claim["exhaustive"]}
257
+ if "table" in claim:
258
+ table, width = claim["table"], 4 if far else 2
259
+ if not isinstance(table, dict) or not isinstance(table.get("evidence"), str) or not table["evidence"].strip():
260
+ raise ValueError("An indirect call table needs layout/count evidence")
261
+ if type(table.get("width", width)) is not int or table.get("width", width) != width:
262
+ raise ValueError(f"An indirect {'far' if far else 'near'} call table holds {width}-byte targets, "
263
+ f"so its width must be {width}")
264
+ layout = _layout(image, table, width, "indirect call table", "indirect call target field offset")
265
+ bound = _bound(image, table, layout, claim["exhaustive"], "indirect call table")
266
+ make_row = _call_row(image, site, far)
267
+ rows = _rows(layout, range(layout[1] or 0), make_row)
268
+ if bound is not None:
269
+ # A call continues at its return site, so its rows never feed its own analysis.
270
+ record, found, graph = derive(image, site, layout, width, bound, [], make_row)
271
+ declaration["derivation"] = record
272
+ if layout[1] is None:
273
+ rows = found or []
274
+ if record["proven"]:
275
+ image.derived_call_reads[site] = (bound["routine"], graph)
276
+ else:
277
+ _check_count(site, layout, record, "indirect call table")
278
+ declaration |= {"table": table, "rows": rows}
279
+ targets = [row["target"] for row in rows]
280
+ else:
281
+ targets = claim["targets"]
282
+ if (not isinstance(targets, list) or not 1 <= len(targets) <= 256
283
+ or len(set(map(repr, targets))) != len(targets)):
284
+ raise ValueError("indirect call targets must be a list of 1..256 distinct file offsets")
285
+ for at in targets:
286
+ integer(at, 0, len(image.data) - 1, "indirect call target")
287
+ region = image.region(at)
288
+ if region is None:
289
+ raise ValueError(f"indirect call target {at} is outside declared code")
290
+ if not far:
291
+ # A near call keeps CS, so its target needs an IP that the call site's mapping places there.
292
+ linear = region["segment"] * 16 + region["ip"] + at - region["start"]
293
+ ip = linear - image.region(site)["segment"] * 16
294
+ if not 0 <= ip <= image.mask or image.near_target(site, ip) != at:
295
+ raise ValueError(f"indirect call target {at} is not placed by any IP in the near call "
296
+ f"site {site}'s segment")
297
+ result[site] = declaration | {"targets": list(dict.fromkeys(targets))}
298
+ return result
299
+
300
+
301
+ def open_reason(declaration, call):
302
+ """Why a declared table or call leaves its site unresolved, or None.
303
+
304
+ A declaration that is not exhaustive may have other targets, and a table without ``count``
305
+ whose count the code did not prove has no rows to follow.
306
+ """
307
+ if not declaration["exhaustive"]:
308
+ return CALL_NOT_EXHAUSTIVE if call else JUMP_NOT_EXHAUSTIVE
309
+ # A derivation still pending (None) has proven nothing yet.
310
+ if "derivation" in declaration and "count" not in declaration["table"] and not (declaration["derivation"] or {}).get("proven"):
311
+ return CALL_NOT_DERIVED if call else JUMP_NOT_DERIVED
312
+ return None
313
+
314
+
315
+ def derived_assumptions(image, indirect_calls):
316
+ """The assumption a report adds when a declared table takes its rows from the code, or an empty list."""
317
+ declarations = [*image.indirect_jumps.values(), *indirect_calls.values()]
318
+ if any("derivation" in d and "count" not in d["table"] for d in declarations):
319
+ return ["each table whose count is derived from the code holds only the rows its derivation gives, "
320
+ "under the assumptions the derivation lists"]
321
+ return []
322
+
323
+
324
+ def declared_call_ends(declaration, no_return_calls):
325
+ """Whether a declared indirect call ends its branch: nothing leaves it unresolved (``open_reason``) and
326
+ every target is in ``no_return_calls``."""
327
+ return open_reason(declaration, True) is None and all(target in no_return_calls
328
+ for target in declaration["targets"])
329
+
330
+
331
+ def call_ends(site, target, indirect_calls, no_return_calls):
332
+ """Whether the call at ``site`` to ``target`` ends its branch, under ``declared_call_ends`` for a declared site."""
333
+ declaration = indirect_calls.get(site)
334
+ if declaration is not None:
335
+ return declared_call_ends(declaration, no_return_calls)
336
+ return target in no_return_calls
@@ -4,7 +4,7 @@ from bisect import bisect_right
4
4
  from pathlib import Path
5
5
  from .image import integer
6
6
  from .trace import LIMIT_REASON, CONTESTED_REASON, OVERLAP_REASON, walk, holding_instruction
7
- from .dispatch import indirect_call_declarations, call_ends
7
+ from .dispatch import indirect_call_declarations, call_ends, derived_assumptions
8
8
 
9
9
  MAX_INVENTORY = 16 * 1024 * 1024
10
10
  MAX_ROWS = 200000
@@ -537,6 +537,7 @@ def inventory_check(image, config):
537
537
  if indirect_calls:
538
538
  assumptions.append("each declared indirect call can call the targets its declaration gives, for the evidence "
539
539
  "it gives, and only those when it is declared exhaustive")
540
+ assumptions += derived_assumptions(image, indirect_calls)
540
541
  indirect_rows = [declaration | {"reached": site in seen} for site, declaration in indirect_calls.items()]
541
542
  return {"inventory": {"path": config["inventory"], "rows": len(rows)},
542
543
  "targets": missing[:limit], "unresolved": unresolved[:limit],
@@ -4,7 +4,7 @@ from .image import integer
4
4
  from .trace import (walk, cfg_step, base_mnemonic, unsupported_transfer, holding_instruction, INTERRUPTS, RETURNS,
5
5
  LIMIT_REASON, OVERLAP_REASON, UNDECODED_REASON)
6
6
  from .pcode_backend import interrupt_vector
7
- from .dispatch import indirect_call_declarations, call_ends
7
+ from .dispatch import indirect_call_declarations, call_ends, derived_assumptions
8
8
 
9
9
  # The virtual root of the dominator computation; no file offset is negative.
10
10
  ROOT = -1
@@ -599,6 +599,7 @@ def reach(image, config):
599
599
  assumptions.append("each declared indirect call can call the targets its declaration gives, for the evidence "
600
600
  "it gives, and only those when it is declared exhaustive")
601
601
  assumptions += no_return_assumptions(no_return_routines, no_return_interrupts, no_return_sites)
602
+ assumptions += derived_assumptions(image, indirect_calls)
602
603
  stopped = stopped or check_stopped
603
604
  # Every range is kept whatever the result limit, so a caller can place any site in the walk; the
604
605
  # instruction limit bounds their number.
@@ -625,7 +626,8 @@ def reach(image, config):
625
626
  "negativeUsable needs a control of either kind, a walk that did not stop at its instruction limit, "
626
627
  "and no unresolved transfer, gap, returnCheckGaps row, contested instruction or contradicted noReturn "
627
628
  "routine, and still rests on the listed assumptions, leaves and noReturn declarations. "
628
- "A declared indirect call counts as resolved when it is declared exhaustive: its table "
629
+ "A declared indirect call counts as resolved when it is declared exhaustive and, for a "
630
+ "table without count, the code proves its count: its table "
629
631
  "rows are read from the build's bytes and its targets list rests on its evidence alone, "
630
632
  "and neither proves an instruction boundary or that the call runs. "
631
633
  "A noReturn routine is contradicted when a return instruction, or a leaf entered "