scientific-method-engine 13.3.1__tar.gz → 13.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/PKG-INFO +2 -2
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/README.md +1 -1
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/pyproject.toml +1 -1
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/cli.py +12 -1
- scientific_method_engine-13.5.0/src/scientific_method_engine/x86/inventory.py +245 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/machine.py +6 -1
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/relational.py +3 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/reports.py +65 -23
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/trace.py +35 -11
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/values.py +19 -5
- scientific_method_engine-13.5.0/tests/test_inventory_check.py +219 -0
- scientific_method_engine-13.5.0/tests/test_term_limit.py +237 -0
- scientific_method_engine-13.3.1/tests/test_term_limit.py +0 -120
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/.gitignore +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/LICENSE +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/__init__.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/__main__.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/__init__.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/image.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/loops.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/pcode.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/pe.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/reach.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/tests/oracle.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/tests/test_continuation_budget.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/tests/test_dispatch.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/tests/test_effect_order.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/tests/test_enter_frames.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/tests/test_entry_frame.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/tests/test_indirect_far_transfers.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/tests/test_interrupt_models.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/tests/test_loops.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/tests/test_memory_scopes.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/tests/test_nested_frame_request.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/tests/test_oracle.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/tests/test_pe.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/tests/test_reach.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/tests/test_relational_controls.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/tests/test_return_checks.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/tests/test_successor_rule.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/tests/test_table_continuations.py +0 -0
- {scientific_method_engine-13.3.1 → scientific_method_engine-13.5.0}/tests/test_x86.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: scientific-method-engine
|
|
3
|
-
Version: 13.
|
|
3
|
+
Version: 13.5.0
|
|
4
4
|
Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
|
|
5
5
|
Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
|
|
6
6
|
Author: kibertoad
|
|
@@ -91,7 +91,7 @@ Exporting for comparison (each writes one file and refuses to overwrite where no
|
|
|
91
91
|
| Script | Arguments | Writes |
|
|
92
92
|
|---|---|---|
|
|
93
93
|
| `ExportBoundedFlow` | entry where an instruction starts, instruction limit (1..10000, the most instruction records exported), output path under `analysis/original/` | instruction metadata of one bounded flow as JSON, with `limitReached` (the walk stopped at the limit with flow left unread) and `noInstruction` (flow targets where no instruction starts). An entry where no instruction starts fails the script and writes nothing |
|
|
94
|
-
| `ExportFunctionInventory` | output TSV path (must not exist) | every function's start and body size |
|
|
94
|
+
| `ExportFunctionInventory` | output TSV path (must not exist) | every function's start and body size, the inventory `inventory-check` compares call targets with |
|
|
95
95
|
| `ExportCallEdges` | output JSON path (must not exist), function limit (1..128), one or more function entries | the call and tail-jump edges of the functions Ghidra reaches breadth-first from the entries, with file offsets, Ghidra's flow type (after any flow override), `fallsThrough` (whether Ghidra continues to the next instruction at the site, after any fall-through override) and `fallsThroughTo`/`fallsThroughToAddress` (where a fall-through override sends Ghidra instead, or null), as the `ghidraCallEdges` input of `callees` |
|
|
96
96
|
| `ExportFunctionFingerprints` | output TSV path (replaced only when the export completes) | per-function and per-instruction fingerprints with addresses normalized, for matching functions across versions |
|
|
97
97
|
|
|
@@ -75,7 +75,7 @@ Exporting for comparison (each writes one file and refuses to overwrite where no
|
|
|
75
75
|
| Script | Arguments | Writes |
|
|
76
76
|
|---|---|---|
|
|
77
77
|
| `ExportBoundedFlow` | entry where an instruction starts, instruction limit (1..10000, the most instruction records exported), output path under `analysis/original/` | instruction metadata of one bounded flow as JSON, with `limitReached` (the walk stopped at the limit with flow left unread) and `noInstruction` (flow targets where no instruction starts). An entry where no instruction starts fails the script and writes nothing |
|
|
78
|
-
| `ExportFunctionInventory` | output TSV path (must not exist) | every function's start and body size |
|
|
78
|
+
| `ExportFunctionInventory` | output TSV path (must not exist) | every function's start and body size, the inventory `inventory-check` compares call targets with |
|
|
79
79
|
| `ExportCallEdges` | output JSON path (must not exist), function limit (1..128), one or more function entries | the call and tail-jump edges of the functions Ghidra reaches breadth-first from the entries, with file offsets, Ghidra's flow type (after any flow override), `fallsThrough` (whether Ghidra continues to the next instruction at the site, after any fall-through override) and `fallsThroughTo`/`fallsThroughToAddress` (where a fall-through override sends Ghidra instead, or null), as the `ghidraCallEdges` input of `callees` |
|
|
80
80
|
| `ExportFunctionFingerprints` | output TSV path (replaced only when the export completes) | per-function and per-instruction fingerprints with addresses normalized, for matching functions across versions |
|
|
81
81
|
|
|
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scientific-method-engine"
|
|
7
7
|
# The release workflow writes the published version from the package's release tag.
|
|
8
|
-
version = "13.
|
|
8
|
+
version = "13.5.0"
|
|
9
9
|
description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
requires-python = ">=3.12"
|
|
@@ -14,7 +14,7 @@ PREPARED_CONFIG_LIMIT = 16 * 1024 * 1024
|
|
|
14
14
|
DECODER = "capstone " + capstone.__version__
|
|
15
15
|
INSTRUCTION_SEMANTICS = f"pypcode {pypcode.__version__} (Ghidra SLEIGH x86)"
|
|
16
16
|
USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bounds|owner|callees|reach|trace|uses|arguments|"
|
|
17
|
-
"effects|returns|memory|incoming|call-order|guards|allocation|dispatch> <config.json|->\n"
|
|
17
|
+
"effects|returns|memory|incoming|inventory-check|call-order|guards|allocation|dispatch> <config.json|->\n"
|
|
18
18
|
"effects includes ordered path writes/calls and local restoration witnesses; transactionality remains unestablished.\n"
|
|
19
19
|
"callees compares its edges with an ExportCallEdges.java export given as ghidraCallEdges;\n"
|
|
20
20
|
"each row it compares fall-through at carries the engine's side as engineReadsOn;\n"
|
|
@@ -24,6 +24,9 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
|
|
|
24
24
|
"reach lists the target sites the starts reach over resolved calls and jumps, with the fewest-call chain\n"
|
|
25
25
|
"and the routines every read route passes, and lists every reached transfer it could not resolve;\n"
|
|
26
26
|
"a leaves routine is reached but not read, and its reason is repeated in the report.\n"
|
|
27
|
+
"inventory-check places every resolved direct call target in the notation of the function inventory TSV\n"
|
|
28
|
+
"that inventory names and lists each target no row starts at: inside another row's body or outside every row,\n"
|
|
29
|
+
"with one calling site, near or far, and whether an entry-path call, a contested one or only raw bytes call it.\n"
|
|
27
30
|
"trace, arguments, effects, returns, guards, memory and allocation check relationalControls:\n"
|
|
28
31
|
"a violated control fails the report; an undecided one is reported and never counts as held.\n"
|
|
29
32
|
"A lastWriter control with an address inspects that memory at its checkpoint anchors without a read.\n"
|
|
@@ -83,6 +86,14 @@ def main(argv):
|
|
|
83
86
|
"scientific-method-engine releases.")
|
|
84
87
|
elif "preparedProtocol" in config:
|
|
85
88
|
raise ValueError("preparedProtocol is set by the reader and cannot be supplied")
|
|
89
|
+
if "inventory" in config:
|
|
90
|
+
inventory = config["inventory"]
|
|
91
|
+
if not isinstance(inventory, str) or not inventory:
|
|
92
|
+
raise ValueError("inventory must name a function inventory TSV file")
|
|
93
|
+
if config_path == "-" and not Path(inventory).is_absolute():
|
|
94
|
+
# The reader resolves inventory against the config's directory; one that does not would send it unresolved.
|
|
95
|
+
raise ValueError("The reader sent a relative inventory path; install a matching @scientific-method/executable-reader")
|
|
96
|
+
config["inventory"] = str(Path(base).absolute() / inventory)
|
|
86
97
|
data, identity = read_source(config, base)
|
|
87
98
|
result = run_report(data, config, command)
|
|
88
99
|
report = {"schema": "bounded-x86-v1", "decoder": DECODER, "instructionSemantics": INSTRUCTION_SEMANTICS,
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
"""Resolved direct call targets compared with a committed function inventory."""
|
|
2
|
+
import re
|
|
3
|
+
from bisect import bisect_right
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from .image import integer
|
|
6
|
+
|
|
7
|
+
MAX_INVENTORY = 16 * 1024 * 1024
|
|
8
|
+
MAX_ROWS = 200000
|
|
9
|
+
COLUMNS = ("start", "size", "name", "out_of_scope", "ranges")
|
|
10
|
+
SEGMENTED = re.compile(r"[0-9A-F]{4}:[0-9A-F]{4}")
|
|
11
|
+
FLAT = re.compile(r"0x[0-9A-F]{8}")
|
|
12
|
+
OFFSET = re.compile(r"0x[0-9A-F]{2,}")
|
|
13
|
+
|
|
14
|
+
START = "inventory start"
|
|
15
|
+
INSIDE = "inside another row's body"
|
|
16
|
+
OUTSIDE = "outside every row"
|
|
17
|
+
UNPLACED = "outside declared code"
|
|
18
|
+
# The evidence of a target, by the rank of its best calling site.
|
|
19
|
+
EVIDENCE = ("entry-path call", "contested call only", "raw byte candidate only")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _place(value, image):
|
|
23
|
+
"""``(space, number)`` for an inventory start or range end, or None when it is not in the notation.
|
|
24
|
+
|
|
25
|
+
Flat32 code is placed by its virtual address. Segmented code is placed by ``segment * 16 + offset``,
|
|
26
|
+
so two spellings of one byte compare equal, or by a ``0x`` file offset, the form the work protocol
|
|
27
|
+
gives overlay code outside the load image.
|
|
28
|
+
"""
|
|
29
|
+
if image.flat:
|
|
30
|
+
return ("address", int(value, 16)) if FLAT.fullmatch(value) else None
|
|
31
|
+
if SEGMENTED.fullmatch(value):
|
|
32
|
+
segment, offset = value.split(":")
|
|
33
|
+
return "address", int(segment, 16) * 16 + int(offset, 16)
|
|
34
|
+
return ("offset", int(value, 16)) if OFFSET.fullmatch(value) else None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def read_inventory(path, image):
|
|
38
|
+
"""The rows of a function inventory TSV (``start``, ``size``, then optionally ``name``, ``out_of_scope``
|
|
39
|
+
and ``ranges``), each with its ``space``, ``at`` and half-open ``body`` ranges. Raises ``ValueError``
|
|
40
|
+
naming the line of the first row that does not parse. ``path`` must be absolute: the CLI resolves a
|
|
41
|
+
config's relative ``inventory`` against the config file's directory, and a library caller resolves
|
|
42
|
+
it the same way, so no path is read against the process's working directory."""
|
|
43
|
+
if not isinstance(path, str) or not path:
|
|
44
|
+
raise ValueError("inventory must name a function inventory TSV file")
|
|
45
|
+
file = Path(path)
|
|
46
|
+
if not file.is_absolute():
|
|
47
|
+
raise ValueError("inventory must be an absolute path; resolve it against the config file's directory")
|
|
48
|
+
if not file.is_file() or file.stat().st_size > MAX_INVENTORY:
|
|
49
|
+
raise ValueError("inventory must be a regular file of at most 16 MiB")
|
|
50
|
+
text = re.sub(r"(?:\r?\n)+$", "", file.read_text(encoding="utf-8").lstrip(""))
|
|
51
|
+
lines = re.split(r"\r?\n", text)
|
|
52
|
+
columns = lines[0].split("\t")
|
|
53
|
+
if columns[:2] != ["start", "size"] or len(set(columns)) != len(columns) or not set(columns) <= set(COLUMNS):
|
|
54
|
+
raise ValueError("inventory columns are start and size, then optionally name, out_of_scope and ranges, each once")
|
|
55
|
+
if len(lines) - 1 > MAX_ROWS:
|
|
56
|
+
raise ValueError(f"inventory holds more than {MAX_ROWS} rows")
|
|
57
|
+
rows, starts = [], set()
|
|
58
|
+
for number, line in enumerate(lines[1:], start=2):
|
|
59
|
+
def bad(why):
|
|
60
|
+
return ValueError(f"inventory line {number}: {why}")
|
|
61
|
+
cells = line.split("\t")
|
|
62
|
+
if len(cells) != len(columns):
|
|
63
|
+
raise bad(f"has {len(cells)} cells, not {len(columns)}")
|
|
64
|
+
cell = {name: value.strip() for name, value in zip(columns, cells)}
|
|
65
|
+
if not re.fullmatch(r"[1-9][0-9]*", cell["size"]):
|
|
66
|
+
raise bad(f"size {cell['size']} is not a positive number of bytes")
|
|
67
|
+
size = int(cell["size"])
|
|
68
|
+
place = _place(cell["start"], image)
|
|
69
|
+
if place is None:
|
|
70
|
+
raise bad(f"start {cell['start']} is not an address in the notation of a "
|
|
71
|
+
f"{'flat32' if image.flat else 'segmented16'} image")
|
|
72
|
+
body = [(place[1], place[1] + size)]
|
|
73
|
+
if cell.get("ranges"):
|
|
74
|
+
body = []
|
|
75
|
+
for item in cell["ranges"].split():
|
|
76
|
+
ends = item.split("..")
|
|
77
|
+
low, high = (_place(e, image) for e in ends) if len(ends) == 2 else (None, None)
|
|
78
|
+
if low is None or high is None or low[0] != place[0] or high[0] != place[0] or high[1] <= low[1]:
|
|
79
|
+
raise bad(f"range {item} is not a half-open range start..end in the notation of its start")
|
|
80
|
+
body.append((low[1], high[1]))
|
|
81
|
+
body.sort()
|
|
82
|
+
if any(body[i][0] < body[i - 1][1] for i in range(1, len(body))):
|
|
83
|
+
raise bad("the ranges overlap")
|
|
84
|
+
if sum(end - begin for begin, end in body) != size:
|
|
85
|
+
raise bad(f"size {size} is not the total of the ranges")
|
|
86
|
+
if not any(begin <= place[1] < end for begin, end in body):
|
|
87
|
+
raise bad(f"start {cell['start']} lies in none of the ranges")
|
|
88
|
+
if place in starts:
|
|
89
|
+
raise bad(f"start {cell['start']} is listed twice")
|
|
90
|
+
starts.add(place)
|
|
91
|
+
row = {"start": cell["start"], "size": size, "space": place[0], "at": place[1], "body": body}
|
|
92
|
+
if cell.get("name"):
|
|
93
|
+
row["name"] = cell["name"]
|
|
94
|
+
rows.append(row)
|
|
95
|
+
return rows
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _target_place(image, at):
|
|
99
|
+
"""``(space, number, written)`` for a call target's file offset, in the notation an inventory uses.
|
|
100
|
+
|
|
101
|
+
Flat32 code is placed by its virtual address, a region with a ``container`` (an overlay view the
|
|
102
|
+
reader declared) by its file offset, and any other segmented region by its ``segment:ip`` mapping.
|
|
103
|
+
None when no declared region holds the target.
|
|
104
|
+
"""
|
|
105
|
+
r = image.region(at)
|
|
106
|
+
if r is None:
|
|
107
|
+
return None
|
|
108
|
+
ip = r["ip"] + at - r["start"]
|
|
109
|
+
if image.flat:
|
|
110
|
+
return "address", ip, f"0x{ip:08X}"
|
|
111
|
+
if r.get("container") is not None:
|
|
112
|
+
return "offset", at, f"0x{at:02X}"
|
|
113
|
+
return "address", r["segment"] * 16 + ip, f"{r['segment']:04X}:{ip:04X}"
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _domains(image):
|
|
117
|
+
"""The ``(space, start, end)`` each declared region covers in an inventory's notation."""
|
|
118
|
+
for r in image.regions:
|
|
119
|
+
kind, low, _ = _target_place(image, r["start"])
|
|
120
|
+
yield kind, low, low + r["end"] - r["start"]
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _count(n, one, many):
|
|
124
|
+
"""``n`` followed by the singular or the plural wording."""
|
|
125
|
+
return f"{n} {one if n == 1 else many}"
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def inventory_check(image, config):
|
|
129
|
+
"""Every resolved direct call target in the searched regions, classified against a function inventory.
|
|
130
|
+
|
|
131
|
+
The calls are those ``incoming`` reads (``direct_calls``). Each distinct target is an inventory
|
|
132
|
+
start, inside the body of another row, outside every row, or outside declared code, and is
|
|
133
|
+
reported with one calling site, near or far, and the evidence of its best site: a call on the
|
|
134
|
+
entry path, a contested instruction, or only a raw byte candidate.
|
|
135
|
+
"""
|
|
136
|
+
from .reports import call_controls, direct_calls, search_coverage
|
|
137
|
+
if "inventory" not in config:
|
|
138
|
+
raise ValueError("inventory-check needs inventory, the path of a function inventory TSV")
|
|
139
|
+
rows = read_inventory(config["inventory"], image)
|
|
140
|
+
limit = integer(config.get("limit", 1000), 1, 10000, "result limit")
|
|
141
|
+
calls = direct_calls(image, config)
|
|
142
|
+
seen, contested = calls["seen"], calls["contested"]
|
|
143
|
+
|
|
144
|
+
starts = {(row["space"], row["at"]) for row in rows}
|
|
145
|
+
pieces = sorted((row["space"], begin, end, i) for i, row in enumerate(rows) for begin, end in row["body"])
|
|
146
|
+
keys = [(space, begin) for space, begin, _, _ in pieces]
|
|
147
|
+
# reach[i] is the furthest end of pieces[0..i] in pieces[i]'s space, so a lookup stops at the first
|
|
148
|
+
# piece before which nothing reaches the target, however long one row is.
|
|
149
|
+
reach = []
|
|
150
|
+
for i, (space, _, end, _) in enumerate(pieces):
|
|
151
|
+
reach.append(max(end, reach[-1]) if i and pieces[i - 1][0] == space else end)
|
|
152
|
+
|
|
153
|
+
def owners(space, at):
|
|
154
|
+
found, i = [], bisect_right(keys, (space, at)) - 1
|
|
155
|
+
while i >= 0 and pieces[i][0] == space and reach[i] > at:
|
|
156
|
+
if pieces[i][1] <= at < pieces[i][2]:
|
|
157
|
+
found.append(pieces[i][3])
|
|
158
|
+
i -= 1
|
|
159
|
+
return sorted(set(found))
|
|
160
|
+
|
|
161
|
+
domains = list(_domains(image))
|
|
162
|
+
outside_code = [row["start"] for row in rows
|
|
163
|
+
if not any(kind == row["space"] and low <= row["at"] < high for kind, low, high in domains)]
|
|
164
|
+
|
|
165
|
+
by_target, unresolved = {}, []
|
|
166
|
+
for row in calls["rows"]:
|
|
167
|
+
if row["target"] is None:
|
|
168
|
+
unresolved.append(row)
|
|
169
|
+
else:
|
|
170
|
+
by_target.setdefault(row["target"], []).append(row)
|
|
171
|
+
|
|
172
|
+
def rank(row):
|
|
173
|
+
return 0 if row["site"] in seen else 1 if row["site"] in contested else 2
|
|
174
|
+
tally = {label: {"targets": 0, "inventoryStarts": 0, "insideAnotherRow": 0, "outsideEveryRow": 0,
|
|
175
|
+
"outsideDeclaredCode": 0} for label in EVIDENCE}
|
|
176
|
+
field = {START: "inventoryStarts", INSIDE: "insideAnotherRow", OUTSIDE: "outsideEveryRow", UNPLACED: "outsideDeclaredCode"}
|
|
177
|
+
missing = []
|
|
178
|
+
for target, sites in sorted(by_target.items()):
|
|
179
|
+
sites.sort(key=lambda row: (rank(row), row["site"]))
|
|
180
|
+
best = sites[0]
|
|
181
|
+
evidence = EVIDENCE[rank(best)]
|
|
182
|
+
placed = _target_place(image, target)
|
|
183
|
+
held = []
|
|
184
|
+
if placed is None:
|
|
185
|
+
status = UNPLACED
|
|
186
|
+
elif placed[:2] in starts:
|
|
187
|
+
status = START
|
|
188
|
+
else:
|
|
189
|
+
held = owners(*placed[:2])
|
|
190
|
+
status = INSIDE if held else OUTSIDE
|
|
191
|
+
tally[evidence]["targets"] += 1
|
|
192
|
+
tally[evidence][field[status]] += 1
|
|
193
|
+
if status == START:
|
|
194
|
+
continue
|
|
195
|
+
entry = {"target": target, "address": placed[2] if placed else None, "status": status, "evidence": evidence,
|
|
196
|
+
"site": best["site"], "call": "far" if best["encoding"] == "lcall" else "near",
|
|
197
|
+
"siteClassification": best["classification"], "region": best["region"], "provenance": best["provenance"],
|
|
198
|
+
"callSites": {"entryPath": sum(rank(r) == 0 for r in sites), "contested": sum(rank(r) == 1 for r in sites),
|
|
199
|
+
"rawCandidates": sum(rank(r) == 2 for r in sites)}}
|
|
200
|
+
if held:
|
|
201
|
+
entry["rows"] = [{k: rows[i][k] for k in ("start", "size", "name") if k in rows[i]} for i in held]
|
|
202
|
+
missing.append(entry)
|
|
203
|
+
|
|
204
|
+
controls = call_controls(calls, config)
|
|
205
|
+
coverage = search_coverage(image, calls["read"])
|
|
206
|
+
partial = any(c["partial"] for c in coverage)
|
|
207
|
+
path = tally["entry-path call"]
|
|
208
|
+
lacking = path["insideAnotherRow"] + path["outsideEveryRow"]
|
|
209
|
+
# A target no declared region places is compared with no row, so it is left out of the comparison's total.
|
|
210
|
+
compared = path["targets"] - path["outsideDeclaredCode"]
|
|
211
|
+
others = sum(t["insideAnotherRow"] + t["outsideEveryRow"] for label, t in tally.items() if label != "entry-path call")
|
|
212
|
+
summary = (f"{lacking} of the {_count(compared, 'call target that entry-path calls resolve to is', 'call targets that entry-path calls resolve to are')} "
|
|
213
|
+
f"not inventory starts: {path['insideAnotherRow']} inside another row's body, "
|
|
214
|
+
f"{path['outsideEveryRow']} outside every row.")
|
|
215
|
+
if path["outsideDeclaredCode"]:
|
|
216
|
+
summary += " " + _count(path["outsideDeclaredCode"],
|
|
217
|
+
"more entry-path call target lies outside declared code and is not compared with the inventory.",
|
|
218
|
+
"more entry-path call targets lie outside declared code and are not compared with the inventory.")
|
|
219
|
+
if others:
|
|
220
|
+
summary += f" {_count(others, 'more comes', 'more come')} only from contested instructions or raw byte candidates."
|
|
221
|
+
if unresolved:
|
|
222
|
+
reached = sum(row["site"] in seen for row in unresolved)
|
|
223
|
+
summary += (f" {_count(len(unresolved), 'call in the searched regions is', 'calls in the searched regions are')} "
|
|
224
|
+
f"unresolved, {reached} of them on the entry path.")
|
|
225
|
+
stopped = {g.get("reason") for g in calls["gaps"]}
|
|
226
|
+
if partial:
|
|
227
|
+
summary += " The search is partial."
|
|
228
|
+
if "instruction limit" in stopped:
|
|
229
|
+
summary += " The entry-path walk stopped at its instruction limit, so it can reach more calls."
|
|
230
|
+
summary += " Calls the search does not resolve can add targets."
|
|
231
|
+
return {"inventory": {"path": config["inventory"], "rows": len(rows)},
|
|
232
|
+
"targets": missing[:limit], "unresolved": unresolved[:limit],
|
|
233
|
+
"rowsOutsideDeclaredCode": outside_code[:limit],
|
|
234
|
+
"counts": {"callTargets": len(by_target), **tally, "unresolvedCalls": len(unresolved),
|
|
235
|
+
"inventoryRows": len(rows), "rowsOutsideDeclaredCode": len(outside_code)},
|
|
236
|
+
"truncated": any(len(x) > limit for x in (missing, unresolved, outside_code)),
|
|
237
|
+
"summary": summary, "controls": controls,
|
|
238
|
+
"searched": [r for r in image.regions if r["name"] in calls["scans"]], "coverage": coverage,
|
|
239
|
+
"partialSearch": partial, "gaps": calls["gaps"],
|
|
240
|
+
"exclusions": ["computed call targets (listed in unresolved when the walk reaches them)", "unrelocated far calls",
|
|
241
|
+
"undeclared mappings", "prefix-started raw candidates off the entry path", "jump targets"],
|
|
242
|
+
"interpretation": "Each distinct resolved direct call target in the searched regions, placed in the inventory's "
|
|
243
|
+
"notation and compared with its rows. A target only raw byte candidates or contested "
|
|
244
|
+
"instructions call is not shown to be code. The counts are lower bounds on the targets "
|
|
245
|
+
"the inventory lacks: calls the search does not resolve can add more."}
|
|
@@ -354,7 +354,12 @@ class State:
|
|
|
354
354
|
def reg(self, name):
|
|
355
355
|
root, low, bits = alias(name)
|
|
356
356
|
value = extract(self.regs[root], low, bits)
|
|
357
|
-
return Value(bits, value.term,
|
|
357
|
+
return Value(bits, value.term, self.reg_origin(name))
|
|
358
|
+
|
|
359
|
+
def reg_origin(self, name):
|
|
360
|
+
"""The sources of register ``name``'s bytes, which ``reg(name)`` carries, without forming its value."""
|
|
361
|
+
root, low, bits = alias(name)
|
|
362
|
+
return tuple(sorted(set().union(*self.reg_sources[root][low // 8:(low + bits) // 8])))
|
|
358
363
|
|
|
359
364
|
def setreg(self, name, value, site):
|
|
360
365
|
root, low, bits = alias(name)
|
|
@@ -432,6 +432,9 @@ def _field(event, path):
|
|
|
432
432
|
raise _Unresolved(f"event {event['kind']} at {event['site']} has no field {path}")
|
|
433
433
|
if not isinstance(value, dict) or "expression" not in value or "bits" not in value:
|
|
434
434
|
raise _Unresolved(f"field {path} of event {event['kind']} at {event['site']} is not a value")
|
|
435
|
+
if value["expression"] is None:
|
|
436
|
+
# A register row the snapshot could not form names why; it has no expression to compare.
|
|
437
|
+
raise _Unresolved(f"field {path} of event {event['kind']} at {event['site']} was not formed: {value['unresolved']}")
|
|
435
438
|
return value
|
|
436
439
|
|
|
437
440
|
|
|
@@ -63,7 +63,7 @@ def search_coverage(image, spans):
|
|
|
63
63
|
holders = [{"view": "section " + section["name"], "start": section["rawStart"],
|
|
64
64
|
"end": section["rawStart"] + section["loadedRawSize"]}]
|
|
65
65
|
if not holders:
|
|
66
|
-
alone.append(r
|
|
66
|
+
alone.append(r)
|
|
67
67
|
for c in holders:
|
|
68
68
|
containers.setdefault((c["view"], c["start"], c["end"]), []).append(r["name"])
|
|
69
69
|
rows = []
|
|
@@ -74,18 +74,31 @@ def search_coverage(image, spans):
|
|
|
74
74
|
"meaning": "partial search: callers in the unsearched ranges are not covered" if missing else
|
|
75
75
|
"the searched regions cover the complete container"})
|
|
76
76
|
if alone:
|
|
77
|
-
|
|
78
|
-
|
|
77
|
+
# With no container to compare with, each region is compared with its own extent, so a scan that
|
|
78
|
+
# scanLimit stopped is reported as partial.
|
|
79
|
+
missing = [gap for r in alone for gap in uncovered(r["start"], r["end"], [spans[r["name"]]])]
|
|
80
|
+
rows.append({"container": None, "regions": [r["name"] for r in alone], "unsearched": missing,
|
|
81
|
+
"partial": bool(missing),
|
|
82
|
+
"meaning": "partial search: no overlay, section or declared segment contains these regions, "
|
|
83
|
+
"and callers in their unsearched ranges are not covered" if missing else
|
|
84
|
+
"no overlay, section or declared segment contains these regions; the search covers them only"})
|
|
79
85
|
return rows
|
|
80
86
|
|
|
81
87
|
|
|
82
|
-
def
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
88
|
+
def direct_calls(image, config):
|
|
89
|
+
"""Every direct call site in the searched regions, as ``incoming`` and ``inventory-check`` read them.
|
|
90
|
+
|
|
91
|
+
Walks the entry-path CFG from every established entry, then scans every byte of the regions
|
|
92
|
+
``searchRegions`` names (all regions by default) for E8 and 9A call starts, and adds each
|
|
93
|
+
reached call the scan cannot see (one that starts with a prefix). Returns a dict with the walk's
|
|
94
|
+
``seen``, ``gaps`` (with a ``raw scan limit`` gap where ``scanLimit`` stopped a region),
|
|
95
|
+
``edges``, ``undecoded`` and ``contested``; ``rows``, every call row in the order it was read,
|
|
96
|
+
each with its ``target`` (None when unresolved); ``scanned``, those rows by site, leaving out
|
|
97
|
+
the reached calls whose frame encoding the walk does not model; ``scans``, the region names
|
|
98
|
+
searched; and ``read``, the byte span the scan read in each.
|
|
99
|
+
"""
|
|
87
100
|
seen, gaps, edges, undecoded, contested = walk(image, entries(image), config.get("instructionLimit", 10000))
|
|
88
|
-
|
|
101
|
+
rows, scanned = [], {}
|
|
89
102
|
|
|
90
103
|
def classify(at):
|
|
91
104
|
if at in seen:
|
|
@@ -117,10 +130,7 @@ def incoming(image, config):
|
|
|
117
130
|
row = {"site": at, "target": resolved, "encoding": ins.mnemonic,
|
|
118
131
|
"classification": classify(at), "provenance": provenance, "region": name}
|
|
119
132
|
scanned[at] = row
|
|
120
|
-
|
|
121
|
-
(hits if at in seen else disputed if at in contested else candidates).append(row)
|
|
122
|
-
elif resolved is None:
|
|
123
|
-
partial.append(row)
|
|
133
|
+
rows.append(row)
|
|
124
134
|
for at, ins in sorted({**seen, **contested}.items()):
|
|
125
135
|
if at in scanned or ins.mnemonic not in ("call", "lcall"):
|
|
126
136
|
continue
|
|
@@ -129,29 +139,52 @@ def incoming(image, config):
|
|
|
129
139
|
continue
|
|
130
140
|
# A reached call can start with a prefix, so the raw E8/9A scan above never sees it.
|
|
131
141
|
if unsupported_transfer(image, ins):
|
|
132
|
-
|
|
133
|
-
|
|
142
|
+
rows.append({"site": at, "target": None, "encoding": ins.mnemonic, "region": region["name"],
|
|
143
|
+
"classification": "unsupported control-transfer frame encoding"})
|
|
134
144
|
continue
|
|
135
145
|
resolved, provenance = call_target(image, at, ins)
|
|
136
146
|
row = {"site": at, "target": resolved, "encoding": ins.mnemonic,
|
|
137
147
|
"classification": classify(at), "provenance": provenance, "region": region["name"]}
|
|
138
148
|
scanned[at] = row
|
|
139
|
-
|
|
140
|
-
(hits if at in seen else disputed).append(row)
|
|
141
|
-
elif resolved is None:
|
|
142
|
-
partial.append(row)
|
|
149
|
+
rows.append(row)
|
|
143
150
|
for edge in edges:
|
|
144
151
|
if edge.get("overlappingTarget") and edge["site"] in scanned:
|
|
145
152
|
scanned[edge["site"]]["overlappingTarget"] = True
|
|
146
153
|
scanned[edge["site"]]["boundaryEvidence"] = edge["boundaryEvidence"]
|
|
147
|
-
|
|
148
|
-
|
|
154
|
+
return {"seen": seen, "gaps": gaps, "edges": edges, "undecoded": undecoded, "contested": contested,
|
|
155
|
+
"rows": rows, "scanned": scanned, "scans": scans, "read": read}
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def call_controls(calls, config):
|
|
159
|
+
"""The rows of the ``controls`` call sites in a ``direct_calls`` result. Raises ``ValueError`` when a
|
|
160
|
+
control is not an entry-path call the scan read with a resolved target."""
|
|
149
161
|
controls = config.get("controls", [])
|
|
150
162
|
if not isinstance(controls, list) or len(controls) > 256:
|
|
151
163
|
raise ValueError("Invalid positive controls")
|
|
164
|
+
scanned, seen = calls["scanned"], calls["seen"]
|
|
152
165
|
for at in controls:
|
|
153
166
|
if type(at) is not int or at not in scanned or at not in seen or scanned[at]["target"] is None:
|
|
154
167
|
raise ValueError(f"Positive control {at} missed or not verified")
|
|
168
|
+
return [scanned[at] for at in controls]
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def incoming(image, config):
|
|
172
|
+
target = integer(config.get("target"), 0, len(image.data) - 1, "target")
|
|
173
|
+
if image.region(target) is None:
|
|
174
|
+
raise ValueError("Incoming target is outside declared code")
|
|
175
|
+
limit = integer(config.get("limit", 100), 1, 10000, "result limit")
|
|
176
|
+
calls = direct_calls(image, config)
|
|
177
|
+
seen, gaps, edges, undecoded, contested = (calls[k] for k in ("seen", "gaps", "edges", "undecoded", "contested"))
|
|
178
|
+
scans, read = calls["scans"], calls["read"]
|
|
179
|
+
hits, candidates, partial, disputed = [], [], [], []
|
|
180
|
+
for row in calls["rows"]:
|
|
181
|
+
if row["target"] is None:
|
|
182
|
+
partial.append(row)
|
|
183
|
+
elif row["target"] == target:
|
|
184
|
+
(hits if row["site"] in seen else disputed if row["site"] in contested else candidates).append(row)
|
|
185
|
+
hits.sort(key=lambda row: row["site"])
|
|
186
|
+
disputed.sort(key=lambda row: row["site"])
|
|
187
|
+
controls = call_controls(calls, config)
|
|
155
188
|
truncated = len(hits) + len(candidates) + len(disputed) + len(partial) > limit
|
|
156
189
|
budget = limit
|
|
157
190
|
def bounded(rows):
|
|
@@ -190,7 +223,7 @@ def incoming(image, config):
|
|
|
190
223
|
return {"target": target, "sections": {k: v[:limit] for k, v in sections.items()}, "confirmed": bounded(hits), "candidates": bounded(candidates),
|
|
191
224
|
"contested": bounded(disputed), "unresolved": bounded(partial),
|
|
192
225
|
"counts": {"confirmed": len(hits), "candidates": len(candidates), "contested": len(disputed), "unresolved": len(partial)},
|
|
193
|
-
"truncated": truncated, "controls":
|
|
226
|
+
"truncated": truncated, "controls": controls,
|
|
194
227
|
"searched": [r for r in image.regions if r["name"] in scans], "coverage": coverage, "partialSearch": partial_scope,
|
|
195
228
|
"unresolvedTransfers": sorted(transfers, key=lambda t: t["site"]), "undecodedRanges": undecoded, "gaps": gaps,
|
|
196
229
|
"negativeUsable": bool(controls) and not (hits or candidates or disputed or partial or gaps or truncated or undecoded or partial_scope),
|
|
@@ -643,7 +676,8 @@ def dispatch(image, config):
|
|
|
643
676
|
outcomes = []
|
|
644
677
|
for path in report["paths"]:
|
|
645
678
|
reached = bool(path["instructionPath"]) and path["instructionPath"][-1] == site
|
|
646
|
-
|
|
679
|
+
index_row = path["registers"].get(index_reg, {})
|
|
680
|
+
index_value = index_row.get("value")
|
|
647
681
|
if reached and index_value is not None:
|
|
648
682
|
index_value *= scale
|
|
649
683
|
if index_value % divisor or index_value // divisor >= count:
|
|
@@ -653,6 +687,9 @@ def dispatch(image, config):
|
|
|
653
687
|
outcomes.append({"status": "selected", "position": index, "rawTarget": rows[index], "encodedIndex": index_value})
|
|
654
688
|
else:
|
|
655
689
|
outcomes.append({"status": "returned-before-dispatch" if path["returned"] else "unresolved", "stop": path["stop"]})
|
|
690
|
+
if reached and "unresolved" in index_row:
|
|
691
|
+
# The site was reached but the index register could not be formed, which the stop alone does not say.
|
|
692
|
+
outcomes[-1]["unresolved"] = index_row["unresolved"]
|
|
656
693
|
outcomes[-1]["transformations"] = [e for e in path["events"] if e["kind"] == "arithmetic"]
|
|
657
694
|
outcomes[-1]["guards"] = path["guards"]
|
|
658
695
|
results.append({"input": value, "outcomes": outcomes, "gaps": report["gaps"]})
|
|
@@ -1718,6 +1755,8 @@ def _run_report(image, config, command):
|
|
|
1718
1755
|
raise ValueError("controlOccurrenceLimit applies only to " + ", ".join(TRACE_COMMANDS))
|
|
1719
1756
|
if "entryFrame" in config and command not in TRACE_COMMANDS:
|
|
1720
1757
|
raise ValueError("entryFrame applies only to " + ", ".join(TRACE_COMMANDS))
|
|
1758
|
+
if "inventory" in config and command != "inventory-check":
|
|
1759
|
+
raise ValueError("inventory applies only to inventory-check")
|
|
1721
1760
|
if command == "operand":
|
|
1722
1761
|
return operand_provenance(image, config)
|
|
1723
1762
|
if command == "target":
|
|
@@ -1735,6 +1774,9 @@ def _run_report(image, config, command):
|
|
|
1735
1774
|
return call_order(image, config)
|
|
1736
1775
|
if command == "incoming":
|
|
1737
1776
|
return incoming(image, config)
|
|
1777
|
+
if command == "inventory-check":
|
|
1778
|
+
from .inventory import inventory_check
|
|
1779
|
+
return inventory_check(image, config)
|
|
1738
1780
|
if command == "operand-candidates":
|
|
1739
1781
|
return operand_candidates(image, config)
|
|
1740
1782
|
if command == "uses":
|
|
@@ -6,7 +6,7 @@ from .image import integer
|
|
|
6
6
|
from .machine import (State, StopPath, REGISTERS, ALIASES, BRANCH_CONDITIONS, string_instruction,
|
|
7
7
|
string_count, string_effect, check_string_form, compare_string, repeated, string_width,
|
|
8
8
|
FLAT_PORT_REASON)
|
|
9
|
-
from .values import const, unknown, sources, op, join, TermLimit
|
|
9
|
+
from .values import const, unknown, sources, op, join, unformed, TermLimit
|
|
10
10
|
from .result_flow import validate_contracts, result_contracts
|
|
11
11
|
from .loops import LoopTracker
|
|
12
12
|
from .memory_scopes import validate_scopes, capture_scopes, retain_scopes, scope_history
|
|
@@ -407,8 +407,18 @@ def hardware_placement(outputs, conditional_outputs, gaps):
|
|
|
407
407
|
return result
|
|
408
408
|
|
|
409
409
|
|
|
410
|
+
def register_row(state, name):
|
|
411
|
+
"""Register ``name``'s report row. A snapshot only observes the path, so a partial register that
|
|
412
|
+
would pass the term limit (its root holds the limit, and selecting the part adds a term) is a row
|
|
413
|
+
with no expression or value and the limit as its ``unresolved`` reason."""
|
|
414
|
+
try:
|
|
415
|
+
return state.reg(name).report()
|
|
416
|
+
except TermLimit as error:
|
|
417
|
+
return unformed(ALIASES[name][2], state.reg_origin(name), str(error))
|
|
418
|
+
|
|
419
|
+
|
|
410
420
|
def snapshot(state):
|
|
411
|
-
return {name: state
|
|
421
|
+
return {name: register_row(state, name) for name in ALIASES}
|
|
412
422
|
|
|
413
423
|
|
|
414
424
|
# The stop for each failed combination of (widthMatches, stackBalanced) at a return.
|
|
@@ -545,18 +555,25 @@ def entry_frame(image, config, entry):
|
|
|
545
555
|
if not arrivals:
|
|
546
556
|
reasons.append(f"no path from {start} reached the entry")
|
|
547
557
|
offsets = sorted({a["sp"] for a in arrivals if a["sp"] is not None})
|
|
548
|
-
|
|
558
|
+
unformed_sp = sorted({a["unresolved"]["sp"] for a in arrivals if "sp" in a.get("unresolved", {})})
|
|
559
|
+
reasons += ["SP at an arrival could not be formed: " + reason for reason in unformed_sp]
|
|
560
|
+
if any(a["sp"] is None and "sp" not in a.get("unresolved", {}) for a in arrivals):
|
|
549
561
|
reasons.append("SP at an arrival is not an offset from the function's entry SP")
|
|
550
562
|
elif len(offsets) > 1:
|
|
551
563
|
reasons.append("arrivals reach the entry with SP at different offsets: " + ", ".join(map(str, offsets)))
|
|
552
564
|
established = not reasons
|
|
553
565
|
bp = {a["bp"] for a in arrivals}
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
566
|
+
frame = {"from": start, "established": established, "sp": offsets[0] if established else None,
|
|
567
|
+
"bp": next(iter(bp)) if established and len(bp) == 1 else None,
|
|
568
|
+
"arrivals": len(arrivals), "pathsRead": len(report["paths"]), "stepsUsed": report["stepsUsed"],
|
|
569
|
+
"reasons": reasons,
|
|
570
|
+
"meaning": "offsets from the entry SP of the function at from, observed at each first arrival on the "
|
|
571
|
+
"traced paths under the query's own inputs; bp null leaves BP unknown"}
|
|
572
|
+
# BP does not decide whether the frame is established, so why it could not be formed has its own field.
|
|
573
|
+
unformed_bp = sorted({a["unresolved"]["bp"] for a in arrivals if "bp" in a.get("unresolved", {})})
|
|
574
|
+
if unformed_bp:
|
|
575
|
+
frame["bpUnresolved"] = unformed_bp
|
|
576
|
+
return frame
|
|
560
577
|
|
|
561
578
|
|
|
562
579
|
def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=None, call_stacks=False, argument_window=0):
|
|
@@ -730,8 +747,15 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
|
|
|
730
747
|
"instructionPath": s.path, "guards": s.guards, "events": s.events,
|
|
731
748
|
"registers": snapshot(s), "conditionalModels": s.conditional}
|
|
732
749
|
if reason == ENTRY_ARRIVAL:
|
|
733
|
-
path["arrival"] = {"frameEntry": s.frames[-1]["entry"]
|
|
734
|
-
|
|
750
|
+
arrival = path["arrival"] = {"frameEntry": s.frames[-1]["entry"]}
|
|
751
|
+
for key, name in (("sp", s.sp), ("bp", s.bp)):
|
|
752
|
+
# Finishing a path only observes it, so a register past the term limit is an offset
|
|
753
|
+
# not known, with the limit as the reason.
|
|
754
|
+
try:
|
|
755
|
+
arrival[key] = s.frame_offset(s.reg(name))
|
|
756
|
+
except TermLimit as error:
|
|
757
|
+
arrival[key] = None
|
|
758
|
+
arrival.setdefault("unresolved", {})[key] = str(error)
|
|
735
759
|
if s.loops is not None:
|
|
736
760
|
path["loops"] = s.loops.report()
|
|
737
761
|
if call_stacks and not returned:
|
|
@@ -40,11 +40,25 @@ class Value:
|
|
|
40
40
|
return self.term[1] if self.term[0] == "constant" else None
|
|
41
41
|
|
|
42
42
|
def report(self):
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
43
|
+
return _row(self.bits, self.term, self.number, self.sources)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def unformed(bits, origin, reason):
|
|
47
|
+
"""The report row of a value the model could not form, with the sources it would have carried.
|
|
48
|
+
|
|
49
|
+
``expression`` and ``value`` are None, ``producers`` and ``resultOrigins`` come from ``origin``
|
|
50
|
+
as ``Value.report`` gives them, and ``unresolved`` says why the value was not formed.
|
|
51
|
+
"""
|
|
52
|
+
return {**_row(bits, None, None, origin), "unresolved": reason}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _row(bits, expression, value, origin):
|
|
56
|
+
"""A report row with the ``producers`` and ``resultOrigins`` that the sources ``origin`` give."""
|
|
57
|
+
row = {"bits": bits, "expression": expression, "value": value, "producers": [s for s in origin if s >= 0]}
|
|
58
|
+
origins = sorted(-s - 1 for s in origin if s < 0)
|
|
59
|
+
if origins:
|
|
60
|
+
row["resultOrigins"] = origins
|
|
61
|
+
return row
|
|
48
62
|
|
|
49
63
|
|
|
50
64
|
def result_marker(order):
|