scientific-method-engine 13.4.0__tar.gz → 13.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/PKG-INFO +2 -2
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/README.md +1 -1
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/pyproject.toml +1 -1
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/cli.py +12 -1
- scientific_method_engine-13.5.0/src/scientific_method_engine/x86/inventory.py +245 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/reports.py +60 -22
- scientific_method_engine-13.5.0/tests/test_inventory_check.py +219 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/.gitignore +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/LICENSE +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/__init__.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/__main__.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/__init__.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/image.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/loops.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/machine.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/pcode.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/pe.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/reach.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/relational.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/trace.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/src/scientific_method_engine/x86/values.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/oracle.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_continuation_budget.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_dispatch.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_effect_order.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_enter_frames.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_entry_frame.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_indirect_far_transfers.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_interrupt_models.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_loops.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_memory_scopes.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_nested_frame_request.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_oracle.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_pe.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_reach.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_relational_controls.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_return_checks.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_successor_rule.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_table_continuations.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_term_limit.py +0 -0
- {scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_x86.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: scientific-method-engine
|
|
3
|
-
Version: 13.
|
|
3
|
+
Version: 13.5.0
|
|
4
4
|
Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
|
|
5
5
|
Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
|
|
6
6
|
Author: kibertoad
|
|
@@ -91,7 +91,7 @@ Exporting for comparison (each writes one file and refuses to overwrite where no
|
|
|
91
91
|
| Script | Arguments | Writes |
|
|
92
92
|
|---|---|---|
|
|
93
93
|
| `ExportBoundedFlow` | entry where an instruction starts, instruction limit (1..10000, the most instruction records exported), output path under `analysis/original/` | instruction metadata of one bounded flow as JSON, with `limitReached` (the walk stopped at the limit with flow left unread) and `noInstruction` (flow targets where no instruction starts). An entry where no instruction starts fails the script and writes nothing |
|
|
94
|
-
| `ExportFunctionInventory` | output TSV path (must not exist) | every function's start and body size |
|
|
94
|
+
| `ExportFunctionInventory` | output TSV path (must not exist) | every function's start and body size, the inventory `inventory-check` compares call targets with |
|
|
95
95
|
| `ExportCallEdges` | output JSON path (must not exist), function limit (1..128), one or more function entries | the call and tail-jump edges of the functions Ghidra reaches breadth-first from the entries, with file offsets, Ghidra's flow type (after any flow override), `fallsThrough` (whether Ghidra continues to the next instruction at the site, after any fall-through override) and `fallsThroughTo`/`fallsThroughToAddress` (where a fall-through override sends Ghidra instead, or null), as the `ghidraCallEdges` input of `callees` |
|
|
96
96
|
| `ExportFunctionFingerprints` | output TSV path (replaced only when the export completes) | per-function and per-instruction fingerprints with addresses normalized, for matching functions across versions |
|
|
97
97
|
|
|
@@ -75,7 +75,7 @@ Exporting for comparison (each writes one file and refuses to overwrite where no
|
|
|
75
75
|
| Script | Arguments | Writes |
|
|
76
76
|
|---|---|---|
|
|
77
77
|
| `ExportBoundedFlow` | entry where an instruction starts, instruction limit (1..10000, the most instruction records exported), output path under `analysis/original/` | instruction metadata of one bounded flow as JSON, with `limitReached` (the walk stopped at the limit with flow left unread) and `noInstruction` (flow targets where no instruction starts). An entry where no instruction starts fails the script and writes nothing |
|
|
78
|
-
| `ExportFunctionInventory` | output TSV path (must not exist) | every function's start and body size |
|
|
78
|
+
| `ExportFunctionInventory` | output TSV path (must not exist) | every function's start and body size, the inventory `inventory-check` compares call targets with |
|
|
79
79
|
| `ExportCallEdges` | output JSON path (must not exist), function limit (1..128), one or more function entries | the call and tail-jump edges of the functions Ghidra reaches breadth-first from the entries, with file offsets, Ghidra's flow type (after any flow override), `fallsThrough` (whether Ghidra continues to the next instruction at the site, after any fall-through override) and `fallsThroughTo`/`fallsThroughToAddress` (where a fall-through override sends Ghidra instead, or null), as the `ghidraCallEdges` input of `callees` |
|
|
80
80
|
| `ExportFunctionFingerprints` | output TSV path (replaced only when the export completes) | per-function and per-instruction fingerprints with addresses normalized, for matching functions across versions |
|
|
81
81
|
|
|
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scientific-method-engine"
|
|
7
7
|
# The release workflow writes the published version from the package's release tag.
|
|
8
|
-
version = "13.
|
|
8
|
+
version = "13.5.0"
|
|
9
9
|
description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
requires-python = ">=3.12"
|
|
@@ -14,7 +14,7 @@ PREPARED_CONFIG_LIMIT = 16 * 1024 * 1024
|
|
|
14
14
|
DECODER = "capstone " + capstone.__version__
|
|
15
15
|
INSTRUCTION_SEMANTICS = f"pypcode {pypcode.__version__} (Ghidra SLEIGH x86)"
|
|
16
16
|
USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bounds|owner|callees|reach|trace|uses|arguments|"
|
|
17
|
-
"effects|returns|memory|incoming|call-order|guards|allocation|dispatch> <config.json|->\n"
|
|
17
|
+
"effects|returns|memory|incoming|inventory-check|call-order|guards|allocation|dispatch> <config.json|->\n"
|
|
18
18
|
"effects includes ordered path writes/calls and local restoration witnesses; transactionality remains unestablished.\n"
|
|
19
19
|
"callees compares its edges with an ExportCallEdges.java export given as ghidraCallEdges;\n"
|
|
20
20
|
"each row it compares fall-through at carries the engine's side as engineReadsOn;\n"
|
|
@@ -24,6 +24,9 @@ USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bou
|
|
|
24
24
|
"reach lists the target sites the starts reach over resolved calls and jumps, with the fewest-call chain\n"
|
|
25
25
|
"and the routines every read route passes, and lists every reached transfer it could not resolve;\n"
|
|
26
26
|
"a leaves routine is reached but not read, and its reason is repeated in the report.\n"
|
|
27
|
+
"inventory-check places every resolved direct call target in the notation of the function inventory TSV\n"
|
|
28
|
+
"that inventory names and lists each target no row starts at: inside another row's body or outside every row,\n"
|
|
29
|
+
"with one calling site, near or far, and whether an entry-path call, a contested one or only raw bytes call it.\n"
|
|
27
30
|
"trace, arguments, effects, returns, guards, memory and allocation check relationalControls:\n"
|
|
28
31
|
"a violated control fails the report; an undecided one is reported and never counts as held.\n"
|
|
29
32
|
"A lastWriter control with an address inspects that memory at its checkpoint anchors without a read.\n"
|
|
@@ -83,6 +86,14 @@ def main(argv):
|
|
|
83
86
|
"scientific-method-engine releases.")
|
|
84
87
|
elif "preparedProtocol" in config:
|
|
85
88
|
raise ValueError("preparedProtocol is set by the reader and cannot be supplied")
|
|
89
|
+
if "inventory" in config:
|
|
90
|
+
inventory = config["inventory"]
|
|
91
|
+
if not isinstance(inventory, str) or not inventory:
|
|
92
|
+
raise ValueError("inventory must name a function inventory TSV file")
|
|
93
|
+
if config_path == "-" and not Path(inventory).is_absolute():
|
|
94
|
+
# The reader resolves inventory against the config's directory; one that does not would send it unresolved.
|
|
95
|
+
raise ValueError("The reader sent a relative inventory path; install a matching @scientific-method/executable-reader")
|
|
96
|
+
config["inventory"] = str(Path(base).absolute() / inventory)
|
|
86
97
|
data, identity = read_source(config, base)
|
|
87
98
|
result = run_report(data, config, command)
|
|
88
99
|
report = {"schema": "bounded-x86-v1", "decoder": DECODER, "instructionSemantics": INSTRUCTION_SEMANTICS,
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
"""Resolved direct call targets compared with a committed function inventory."""
|
|
2
|
+
import re
|
|
3
|
+
from bisect import bisect_right
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from .image import integer
|
|
6
|
+
|
|
7
|
+
MAX_INVENTORY = 16 * 1024 * 1024
|
|
8
|
+
MAX_ROWS = 200000
|
|
9
|
+
COLUMNS = ("start", "size", "name", "out_of_scope", "ranges")
|
|
10
|
+
SEGMENTED = re.compile(r"[0-9A-F]{4}:[0-9A-F]{4}")
|
|
11
|
+
FLAT = re.compile(r"0x[0-9A-F]{8}")
|
|
12
|
+
OFFSET = re.compile(r"0x[0-9A-F]{2,}")
|
|
13
|
+
|
|
14
|
+
START = "inventory start"
|
|
15
|
+
INSIDE = "inside another row's body"
|
|
16
|
+
OUTSIDE = "outside every row"
|
|
17
|
+
UNPLACED = "outside declared code"
|
|
18
|
+
# The evidence of a target, by the rank of its best calling site.
|
|
19
|
+
EVIDENCE = ("entry-path call", "contested call only", "raw byte candidate only")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _place(value, image):
|
|
23
|
+
"""``(space, number)`` for an inventory start or range end, or None when it is not in the notation.
|
|
24
|
+
|
|
25
|
+
Flat32 code is placed by its virtual address. Segmented code is placed by ``segment * 16 + offset``,
|
|
26
|
+
so two spellings of one byte compare equal, or by a ``0x`` file offset, the form the work protocol
|
|
27
|
+
gives overlay code outside the load image.
|
|
28
|
+
"""
|
|
29
|
+
if image.flat:
|
|
30
|
+
return ("address", int(value, 16)) if FLAT.fullmatch(value) else None
|
|
31
|
+
if SEGMENTED.fullmatch(value):
|
|
32
|
+
segment, offset = value.split(":")
|
|
33
|
+
return "address", int(segment, 16) * 16 + int(offset, 16)
|
|
34
|
+
return ("offset", int(value, 16)) if OFFSET.fullmatch(value) else None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def read_inventory(path, image):
|
|
38
|
+
"""The rows of a function inventory TSV (``start``, ``size``, then optionally ``name``, ``out_of_scope``
|
|
39
|
+
and ``ranges``), each with its ``space``, ``at`` and half-open ``body`` ranges. Raises ``ValueError``
|
|
40
|
+
naming the line of the first row that does not parse. ``path`` must be absolute: the CLI resolves a
|
|
41
|
+
config's relative ``inventory`` against the config file's directory, and a library caller resolves
|
|
42
|
+
it the same way, so no path is read against the process's working directory."""
|
|
43
|
+
if not isinstance(path, str) or not path:
|
|
44
|
+
raise ValueError("inventory must name a function inventory TSV file")
|
|
45
|
+
file = Path(path)
|
|
46
|
+
if not file.is_absolute():
|
|
47
|
+
raise ValueError("inventory must be an absolute path; resolve it against the config file's directory")
|
|
48
|
+
if not file.is_file() or file.stat().st_size > MAX_INVENTORY:
|
|
49
|
+
raise ValueError("inventory must be a regular file of at most 16 MiB")
|
|
50
|
+
text = re.sub(r"(?:\r?\n)+$", "", file.read_text(encoding="utf-8").lstrip(""))
|
|
51
|
+
lines = re.split(r"\r?\n", text)
|
|
52
|
+
columns = lines[0].split("\t")
|
|
53
|
+
if columns[:2] != ["start", "size"] or len(set(columns)) != len(columns) or not set(columns) <= set(COLUMNS):
|
|
54
|
+
raise ValueError("inventory columns are start and size, then optionally name, out_of_scope and ranges, each once")
|
|
55
|
+
if len(lines) - 1 > MAX_ROWS:
|
|
56
|
+
raise ValueError(f"inventory holds more than {MAX_ROWS} rows")
|
|
57
|
+
rows, starts = [], set()
|
|
58
|
+
for number, line in enumerate(lines[1:], start=2):
|
|
59
|
+
def bad(why):
|
|
60
|
+
return ValueError(f"inventory line {number}: {why}")
|
|
61
|
+
cells = line.split("\t")
|
|
62
|
+
if len(cells) != len(columns):
|
|
63
|
+
raise bad(f"has {len(cells)} cells, not {len(columns)}")
|
|
64
|
+
cell = {name: value.strip() for name, value in zip(columns, cells)}
|
|
65
|
+
if not re.fullmatch(r"[1-9][0-9]*", cell["size"]):
|
|
66
|
+
raise bad(f"size {cell['size']} is not a positive number of bytes")
|
|
67
|
+
size = int(cell["size"])
|
|
68
|
+
place = _place(cell["start"], image)
|
|
69
|
+
if place is None:
|
|
70
|
+
raise bad(f"start {cell['start']} is not an address in the notation of a "
|
|
71
|
+
f"{'flat32' if image.flat else 'segmented16'} image")
|
|
72
|
+
body = [(place[1], place[1] + size)]
|
|
73
|
+
if cell.get("ranges"):
|
|
74
|
+
body = []
|
|
75
|
+
for item in cell["ranges"].split():
|
|
76
|
+
ends = item.split("..")
|
|
77
|
+
low, high = (_place(e, image) for e in ends) if len(ends) == 2 else (None, None)
|
|
78
|
+
if low is None or high is None or low[0] != place[0] or high[0] != place[0] or high[1] <= low[1]:
|
|
79
|
+
raise bad(f"range {item} is not a half-open range start..end in the notation of its start")
|
|
80
|
+
body.append((low[1], high[1]))
|
|
81
|
+
body.sort()
|
|
82
|
+
if any(body[i][0] < body[i - 1][1] for i in range(1, len(body))):
|
|
83
|
+
raise bad("the ranges overlap")
|
|
84
|
+
if sum(end - begin for begin, end in body) != size:
|
|
85
|
+
raise bad(f"size {size} is not the total of the ranges")
|
|
86
|
+
if not any(begin <= place[1] < end for begin, end in body):
|
|
87
|
+
raise bad(f"start {cell['start']} lies in none of the ranges")
|
|
88
|
+
if place in starts:
|
|
89
|
+
raise bad(f"start {cell['start']} is listed twice")
|
|
90
|
+
starts.add(place)
|
|
91
|
+
row = {"start": cell["start"], "size": size, "space": place[0], "at": place[1], "body": body}
|
|
92
|
+
if cell.get("name"):
|
|
93
|
+
row["name"] = cell["name"]
|
|
94
|
+
rows.append(row)
|
|
95
|
+
return rows
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _target_place(image, at):
|
|
99
|
+
"""``(space, number, written)`` for a call target's file offset, in the notation an inventory uses.
|
|
100
|
+
|
|
101
|
+
Flat32 code is placed by its virtual address, a region with a ``container`` (an overlay view the
|
|
102
|
+
reader declared) by its file offset, and any other segmented region by its ``segment:ip`` mapping.
|
|
103
|
+
None when no declared region holds the target.
|
|
104
|
+
"""
|
|
105
|
+
r = image.region(at)
|
|
106
|
+
if r is None:
|
|
107
|
+
return None
|
|
108
|
+
ip = r["ip"] + at - r["start"]
|
|
109
|
+
if image.flat:
|
|
110
|
+
return "address", ip, f"0x{ip:08X}"
|
|
111
|
+
if r.get("container") is not None:
|
|
112
|
+
return "offset", at, f"0x{at:02X}"
|
|
113
|
+
return "address", r["segment"] * 16 + ip, f"{r['segment']:04X}:{ip:04X}"
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _domains(image):
|
|
117
|
+
"""The ``(space, start, end)`` each declared region covers in an inventory's notation."""
|
|
118
|
+
for r in image.regions:
|
|
119
|
+
kind, low, _ = _target_place(image, r["start"])
|
|
120
|
+
yield kind, low, low + r["end"] - r["start"]
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _count(n, one, many):
|
|
124
|
+
"""``n`` followed by the singular or the plural wording."""
|
|
125
|
+
return f"{n} {one if n == 1 else many}"
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def inventory_check(image, config):
|
|
129
|
+
"""Every resolved direct call target in the searched regions, classified against a function inventory.
|
|
130
|
+
|
|
131
|
+
The calls are those ``incoming`` reads (``direct_calls``). Each distinct target is an inventory
|
|
132
|
+
start, inside the body of another row, outside every row, or outside declared code, and is
|
|
133
|
+
reported with one calling site, near or far, and the evidence of its best site: a call on the
|
|
134
|
+
entry path, a contested instruction, or only a raw byte candidate.
|
|
135
|
+
"""
|
|
136
|
+
from .reports import call_controls, direct_calls, search_coverage
|
|
137
|
+
if "inventory" not in config:
|
|
138
|
+
raise ValueError("inventory-check needs inventory, the path of a function inventory TSV")
|
|
139
|
+
rows = read_inventory(config["inventory"], image)
|
|
140
|
+
limit = integer(config.get("limit", 1000), 1, 10000, "result limit")
|
|
141
|
+
calls = direct_calls(image, config)
|
|
142
|
+
seen, contested = calls["seen"], calls["contested"]
|
|
143
|
+
|
|
144
|
+
starts = {(row["space"], row["at"]) for row in rows}
|
|
145
|
+
pieces = sorted((row["space"], begin, end, i) for i, row in enumerate(rows) for begin, end in row["body"])
|
|
146
|
+
keys = [(space, begin) for space, begin, _, _ in pieces]
|
|
147
|
+
# reach[i] is the furthest end of pieces[0..i] in pieces[i]'s space, so a lookup stops at the first
|
|
148
|
+
# piece before which nothing reaches the target, however long one row is.
|
|
149
|
+
reach = []
|
|
150
|
+
for i, (space, _, end, _) in enumerate(pieces):
|
|
151
|
+
reach.append(max(end, reach[-1]) if i and pieces[i - 1][0] == space else end)
|
|
152
|
+
|
|
153
|
+
def owners(space, at):
|
|
154
|
+
found, i = [], bisect_right(keys, (space, at)) - 1
|
|
155
|
+
while i >= 0 and pieces[i][0] == space and reach[i] > at:
|
|
156
|
+
if pieces[i][1] <= at < pieces[i][2]:
|
|
157
|
+
found.append(pieces[i][3])
|
|
158
|
+
i -= 1
|
|
159
|
+
return sorted(set(found))
|
|
160
|
+
|
|
161
|
+
domains = list(_domains(image))
|
|
162
|
+
outside_code = [row["start"] for row in rows
|
|
163
|
+
if not any(kind == row["space"] and low <= row["at"] < high for kind, low, high in domains)]
|
|
164
|
+
|
|
165
|
+
by_target, unresolved = {}, []
|
|
166
|
+
for row in calls["rows"]:
|
|
167
|
+
if row["target"] is None:
|
|
168
|
+
unresolved.append(row)
|
|
169
|
+
else:
|
|
170
|
+
by_target.setdefault(row["target"], []).append(row)
|
|
171
|
+
|
|
172
|
+
def rank(row):
|
|
173
|
+
return 0 if row["site"] in seen else 1 if row["site"] in contested else 2
|
|
174
|
+
tally = {label: {"targets": 0, "inventoryStarts": 0, "insideAnotherRow": 0, "outsideEveryRow": 0,
|
|
175
|
+
"outsideDeclaredCode": 0} for label in EVIDENCE}
|
|
176
|
+
field = {START: "inventoryStarts", INSIDE: "insideAnotherRow", OUTSIDE: "outsideEveryRow", UNPLACED: "outsideDeclaredCode"}
|
|
177
|
+
missing = []
|
|
178
|
+
for target, sites in sorted(by_target.items()):
|
|
179
|
+
sites.sort(key=lambda row: (rank(row), row["site"]))
|
|
180
|
+
best = sites[0]
|
|
181
|
+
evidence = EVIDENCE[rank(best)]
|
|
182
|
+
placed = _target_place(image, target)
|
|
183
|
+
held = []
|
|
184
|
+
if placed is None:
|
|
185
|
+
status = UNPLACED
|
|
186
|
+
elif placed[:2] in starts:
|
|
187
|
+
status = START
|
|
188
|
+
else:
|
|
189
|
+
held = owners(*placed[:2])
|
|
190
|
+
status = INSIDE if held else OUTSIDE
|
|
191
|
+
tally[evidence]["targets"] += 1
|
|
192
|
+
tally[evidence][field[status]] += 1
|
|
193
|
+
if status == START:
|
|
194
|
+
continue
|
|
195
|
+
entry = {"target": target, "address": placed[2] if placed else None, "status": status, "evidence": evidence,
|
|
196
|
+
"site": best["site"], "call": "far" if best["encoding"] == "lcall" else "near",
|
|
197
|
+
"siteClassification": best["classification"], "region": best["region"], "provenance": best["provenance"],
|
|
198
|
+
"callSites": {"entryPath": sum(rank(r) == 0 for r in sites), "contested": sum(rank(r) == 1 for r in sites),
|
|
199
|
+
"rawCandidates": sum(rank(r) == 2 for r in sites)}}
|
|
200
|
+
if held:
|
|
201
|
+
entry["rows"] = [{k: rows[i][k] for k in ("start", "size", "name") if k in rows[i]} for i in held]
|
|
202
|
+
missing.append(entry)
|
|
203
|
+
|
|
204
|
+
controls = call_controls(calls, config)
|
|
205
|
+
coverage = search_coverage(image, calls["read"])
|
|
206
|
+
partial = any(c["partial"] for c in coverage)
|
|
207
|
+
path = tally["entry-path call"]
|
|
208
|
+
lacking = path["insideAnotherRow"] + path["outsideEveryRow"]
|
|
209
|
+
# A target no declared region places is compared with no row, so it is left out of the comparison's total.
|
|
210
|
+
compared = path["targets"] - path["outsideDeclaredCode"]
|
|
211
|
+
others = sum(t["insideAnotherRow"] + t["outsideEveryRow"] for label, t in tally.items() if label != "entry-path call")
|
|
212
|
+
summary = (f"{lacking} of the {_count(compared, 'call target that entry-path calls resolve to is', 'call targets that entry-path calls resolve to are')} "
|
|
213
|
+
f"not inventory starts: {path['insideAnotherRow']} inside another row's body, "
|
|
214
|
+
f"{path['outsideEveryRow']} outside every row.")
|
|
215
|
+
if path["outsideDeclaredCode"]:
|
|
216
|
+
summary += " " + _count(path["outsideDeclaredCode"],
|
|
217
|
+
"more entry-path call target lies outside declared code and is not compared with the inventory.",
|
|
218
|
+
"more entry-path call targets lie outside declared code and are not compared with the inventory.")
|
|
219
|
+
if others:
|
|
220
|
+
summary += f" {_count(others, 'more comes', 'more come')} only from contested instructions or raw byte candidates."
|
|
221
|
+
if unresolved:
|
|
222
|
+
reached = sum(row["site"] in seen for row in unresolved)
|
|
223
|
+
summary += (f" {_count(len(unresolved), 'call in the searched regions is', 'calls in the searched regions are')} "
|
|
224
|
+
f"unresolved, {reached} of them on the entry path.")
|
|
225
|
+
stopped = {g.get("reason") for g in calls["gaps"]}
|
|
226
|
+
if partial:
|
|
227
|
+
summary += " The search is partial."
|
|
228
|
+
if "instruction limit" in stopped:
|
|
229
|
+
summary += " The entry-path walk stopped at its instruction limit, so it can reach more calls."
|
|
230
|
+
summary += " Calls the search does not resolve can add targets."
|
|
231
|
+
return {"inventory": {"path": config["inventory"], "rows": len(rows)},
|
|
232
|
+
"targets": missing[:limit], "unresolved": unresolved[:limit],
|
|
233
|
+
"rowsOutsideDeclaredCode": outside_code[:limit],
|
|
234
|
+
"counts": {"callTargets": len(by_target), **tally, "unresolvedCalls": len(unresolved),
|
|
235
|
+
"inventoryRows": len(rows), "rowsOutsideDeclaredCode": len(outside_code)},
|
|
236
|
+
"truncated": any(len(x) > limit for x in (missing, unresolved, outside_code)),
|
|
237
|
+
"summary": summary, "controls": controls,
|
|
238
|
+
"searched": [r for r in image.regions if r["name"] in calls["scans"]], "coverage": coverage,
|
|
239
|
+
"partialSearch": partial, "gaps": calls["gaps"],
|
|
240
|
+
"exclusions": ["computed call targets (listed in unresolved when the walk reaches them)", "unrelocated far calls",
|
|
241
|
+
"undeclared mappings", "prefix-started raw candidates off the entry path", "jump targets"],
|
|
242
|
+
"interpretation": "Each distinct resolved direct call target in the searched regions, placed in the inventory's "
|
|
243
|
+
"notation and compared with its rows. A target only raw byte candidates or contested "
|
|
244
|
+
"instructions call is not shown to be code. The counts are lower bounds on the targets "
|
|
245
|
+
"the inventory lacks: calls the search does not resolve can add more."}
|
|
@@ -63,7 +63,7 @@ def search_coverage(image, spans):
|
|
|
63
63
|
holders = [{"view": "section " + section["name"], "start": section["rawStart"],
|
|
64
64
|
"end": section["rawStart"] + section["loadedRawSize"]}]
|
|
65
65
|
if not holders:
|
|
66
|
-
alone.append(r
|
|
66
|
+
alone.append(r)
|
|
67
67
|
for c in holders:
|
|
68
68
|
containers.setdefault((c["view"], c["start"], c["end"]), []).append(r["name"])
|
|
69
69
|
rows = []
|
|
@@ -74,18 +74,31 @@ def search_coverage(image, spans):
|
|
|
74
74
|
"meaning": "partial search: callers in the unsearched ranges are not covered" if missing else
|
|
75
75
|
"the searched regions cover the complete container"})
|
|
76
76
|
if alone:
|
|
77
|
-
|
|
78
|
-
|
|
77
|
+
# With no container to compare with, each region is compared with its own extent, so a scan that
|
|
78
|
+
# scanLimit stopped is reported as partial.
|
|
79
|
+
missing = [gap for r in alone for gap in uncovered(r["start"], r["end"], [spans[r["name"]]])]
|
|
80
|
+
rows.append({"container": None, "regions": [r["name"] for r in alone], "unsearched": missing,
|
|
81
|
+
"partial": bool(missing),
|
|
82
|
+
"meaning": "partial search: no overlay, section or declared segment contains these regions, "
|
|
83
|
+
"and callers in their unsearched ranges are not covered" if missing else
|
|
84
|
+
"no overlay, section or declared segment contains these regions; the search covers them only"})
|
|
79
85
|
return rows
|
|
80
86
|
|
|
81
87
|
|
|
82
|
-
def
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
88
|
+
def direct_calls(image, config):
|
|
89
|
+
"""Every direct call site in the searched regions, as ``incoming`` and ``inventory-check`` read them.
|
|
90
|
+
|
|
91
|
+
Walks the entry-path CFG from every established entry, then scans every byte of the regions
|
|
92
|
+
``searchRegions`` names (all regions by default) for E8 and 9A call starts, and adds each
|
|
93
|
+
reached call the scan cannot see (one that starts with a prefix). Returns a dict with the walk's
|
|
94
|
+
``seen``, ``gaps`` (with a ``raw scan limit`` gap where ``scanLimit`` stopped a region),
|
|
95
|
+
``edges``, ``undecoded`` and ``contested``; ``rows``, every call row in the order it was read,
|
|
96
|
+
each with its ``target`` (None when unresolved); ``scanned``, those rows by site, leaving out
|
|
97
|
+
the reached calls whose frame encoding the walk does not model; ``scans``, the region names
|
|
98
|
+
searched; and ``read``, the byte span the scan read in each.
|
|
99
|
+
"""
|
|
87
100
|
seen, gaps, edges, undecoded, contested = walk(image, entries(image), config.get("instructionLimit", 10000))
|
|
88
|
-
|
|
101
|
+
rows, scanned = [], {}
|
|
89
102
|
|
|
90
103
|
def classify(at):
|
|
91
104
|
if at in seen:
|
|
@@ -117,10 +130,7 @@ def incoming(image, config):
|
|
|
117
130
|
row = {"site": at, "target": resolved, "encoding": ins.mnemonic,
|
|
118
131
|
"classification": classify(at), "provenance": provenance, "region": name}
|
|
119
132
|
scanned[at] = row
|
|
120
|
-
|
|
121
|
-
(hits if at in seen else disputed if at in contested else candidates).append(row)
|
|
122
|
-
elif resolved is None:
|
|
123
|
-
partial.append(row)
|
|
133
|
+
rows.append(row)
|
|
124
134
|
for at, ins in sorted({**seen, **contested}.items()):
|
|
125
135
|
if at in scanned or ins.mnemonic not in ("call", "lcall"):
|
|
126
136
|
continue
|
|
@@ -129,29 +139,52 @@ def incoming(image, config):
|
|
|
129
139
|
continue
|
|
130
140
|
# A reached call can start with a prefix, so the raw E8/9A scan above never sees it.
|
|
131
141
|
if unsupported_transfer(image, ins):
|
|
132
|
-
|
|
133
|
-
|
|
142
|
+
rows.append({"site": at, "target": None, "encoding": ins.mnemonic, "region": region["name"],
|
|
143
|
+
"classification": "unsupported control-transfer frame encoding"})
|
|
134
144
|
continue
|
|
135
145
|
resolved, provenance = call_target(image, at, ins)
|
|
136
146
|
row = {"site": at, "target": resolved, "encoding": ins.mnemonic,
|
|
137
147
|
"classification": classify(at), "provenance": provenance, "region": region["name"]}
|
|
138
148
|
scanned[at] = row
|
|
139
|
-
|
|
140
|
-
(hits if at in seen else disputed).append(row)
|
|
141
|
-
elif resolved is None:
|
|
142
|
-
partial.append(row)
|
|
149
|
+
rows.append(row)
|
|
143
150
|
for edge in edges:
|
|
144
151
|
if edge.get("overlappingTarget") and edge["site"] in scanned:
|
|
145
152
|
scanned[edge["site"]]["overlappingTarget"] = True
|
|
146
153
|
scanned[edge["site"]]["boundaryEvidence"] = edge["boundaryEvidence"]
|
|
147
|
-
|
|
148
|
-
|
|
154
|
+
return {"seen": seen, "gaps": gaps, "edges": edges, "undecoded": undecoded, "contested": contested,
|
|
155
|
+
"rows": rows, "scanned": scanned, "scans": scans, "read": read}
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def call_controls(calls, config):
|
|
159
|
+
"""The rows of the ``controls`` call sites in a ``direct_calls`` result. Raises ``ValueError`` when a
|
|
160
|
+
control is not an entry-path call the scan read with a resolved target."""
|
|
149
161
|
controls = config.get("controls", [])
|
|
150
162
|
if not isinstance(controls, list) or len(controls) > 256:
|
|
151
163
|
raise ValueError("Invalid positive controls")
|
|
164
|
+
scanned, seen = calls["scanned"], calls["seen"]
|
|
152
165
|
for at in controls:
|
|
153
166
|
if type(at) is not int or at not in scanned or at not in seen or scanned[at]["target"] is None:
|
|
154
167
|
raise ValueError(f"Positive control {at} missed or not verified")
|
|
168
|
+
return [scanned[at] for at in controls]
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def incoming(image, config):
|
|
172
|
+
target = integer(config.get("target"), 0, len(image.data) - 1, "target")
|
|
173
|
+
if image.region(target) is None:
|
|
174
|
+
raise ValueError("Incoming target is outside declared code")
|
|
175
|
+
limit = integer(config.get("limit", 100), 1, 10000, "result limit")
|
|
176
|
+
calls = direct_calls(image, config)
|
|
177
|
+
seen, gaps, edges, undecoded, contested = (calls[k] for k in ("seen", "gaps", "edges", "undecoded", "contested"))
|
|
178
|
+
scans, read = calls["scans"], calls["read"]
|
|
179
|
+
hits, candidates, partial, disputed = [], [], [], []
|
|
180
|
+
for row in calls["rows"]:
|
|
181
|
+
if row["target"] is None:
|
|
182
|
+
partial.append(row)
|
|
183
|
+
elif row["target"] == target:
|
|
184
|
+
(hits if row["site"] in seen else disputed if row["site"] in contested else candidates).append(row)
|
|
185
|
+
hits.sort(key=lambda row: row["site"])
|
|
186
|
+
disputed.sort(key=lambda row: row["site"])
|
|
187
|
+
controls = call_controls(calls, config)
|
|
155
188
|
truncated = len(hits) + len(candidates) + len(disputed) + len(partial) > limit
|
|
156
189
|
budget = limit
|
|
157
190
|
def bounded(rows):
|
|
@@ -190,7 +223,7 @@ def incoming(image, config):
|
|
|
190
223
|
return {"target": target, "sections": {k: v[:limit] for k, v in sections.items()}, "confirmed": bounded(hits), "candidates": bounded(candidates),
|
|
191
224
|
"contested": bounded(disputed), "unresolved": bounded(partial),
|
|
192
225
|
"counts": {"confirmed": len(hits), "candidates": len(candidates), "contested": len(disputed), "unresolved": len(partial)},
|
|
193
|
-
"truncated": truncated, "controls":
|
|
226
|
+
"truncated": truncated, "controls": controls,
|
|
194
227
|
"searched": [r for r in image.regions if r["name"] in scans], "coverage": coverage, "partialSearch": partial_scope,
|
|
195
228
|
"unresolvedTransfers": sorted(transfers, key=lambda t: t["site"]), "undecodedRanges": undecoded, "gaps": gaps,
|
|
196
229
|
"negativeUsable": bool(controls) and not (hits or candidates or disputed or partial or gaps or truncated or undecoded or partial_scope),
|
|
@@ -1722,6 +1755,8 @@ def _run_report(image, config, command):
|
|
|
1722
1755
|
raise ValueError("controlOccurrenceLimit applies only to " + ", ".join(TRACE_COMMANDS))
|
|
1723
1756
|
if "entryFrame" in config and command not in TRACE_COMMANDS:
|
|
1724
1757
|
raise ValueError("entryFrame applies only to " + ", ".join(TRACE_COMMANDS))
|
|
1758
|
+
if "inventory" in config and command != "inventory-check":
|
|
1759
|
+
raise ValueError("inventory applies only to inventory-check")
|
|
1725
1760
|
if command == "operand":
|
|
1726
1761
|
return operand_provenance(image, config)
|
|
1727
1762
|
if command == "target":
|
|
@@ -1739,6 +1774,9 @@ def _run_report(image, config, command):
|
|
|
1739
1774
|
return call_order(image, config)
|
|
1740
1775
|
if command == "incoming":
|
|
1741
1776
|
return incoming(image, config)
|
|
1777
|
+
if command == "inventory-check":
|
|
1778
|
+
from .inventory import inventory_check
|
|
1779
|
+
return inventory_check(image, config)
|
|
1742
1780
|
if command == "operand-candidates":
|
|
1743
1781
|
return operand_candidates(image, config)
|
|
1744
1782
|
if command == "uses":
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"""Synthetic machine code and synthetic inventories only. No original binaries or analysis artifacts."""
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
import subprocess
|
|
6
|
+
import sys
|
|
7
|
+
import tempfile
|
|
8
|
+
import unittest
|
|
9
|
+
|
|
10
|
+
SRC = Path(__file__).resolve().parents[1] / "src"
|
|
11
|
+
sys.path.insert(0, str(SRC))
|
|
12
|
+
import xxhash
|
|
13
|
+
from scientific_method_engine.x86.reports import run_report
|
|
14
|
+
|
|
15
|
+
ENGINE = [sys.executable, "-B", "-m", "scientific_method_engine"]
|
|
16
|
+
ENGINE_ENV = {**os.environ, "PYTHONPATH": os.pathsep.join(filter(None, [str(SRC), os.environ.get("PYTHONPATH")]))}
|
|
17
|
+
|
|
18
|
+
# One resident segment at 1000h. 0000 calls 0006, 0003 calls 000B; 0007..000A is padding.
|
|
19
|
+
CALLS = bytes.fromhex(
|
|
20
|
+
"e80300" # 0000 call 0006
|
|
21
|
+
"e80500" # 0003 call 000B
|
|
22
|
+
"c3" # 0006 ret
|
|
23
|
+
"90909090" # 0007 padding
|
|
24
|
+
"c3" # 000B ret
|
|
25
|
+
)
|
|
26
|
+
# CALLS, then bytes no entry path reaches: 000C calls 000F, which no inventory row starts at.
|
|
27
|
+
RAW = CALLS + bytes.fromhex("e80000" "c3")
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def config(data=CALLS, regions=None, **extra):
|
|
31
|
+
regions = regions or [{"name": "resident", "start": 0, "end": len(data), "ip": 0, "segment": 0x1000,
|
|
32
|
+
"resident": True, "entries": [0], "evidence": "synthetic declared code extent"}]
|
|
33
|
+
return {"regions": regions, **extra}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class InventoryCheckTests(unittest.TestCase):
|
|
37
|
+
def setUp(self):
|
|
38
|
+
folder = tempfile.TemporaryDirectory()
|
|
39
|
+
self.addCleanup(folder.cleanup)
|
|
40
|
+
self.root = Path(folder.name)
|
|
41
|
+
|
|
42
|
+
def inventory(self, text, name="inventory.tsv"):
|
|
43
|
+
path = self.root / name
|
|
44
|
+
path.write_text(text, encoding="utf-8", newline="")
|
|
45
|
+
return str(path)
|
|
46
|
+
|
|
47
|
+
def check(self, text, data=CALLS, **extra):
|
|
48
|
+
return run_report(data, config(data, inventory=self.inventory(text), **extra), "inventory-check")
|
|
49
|
+
|
|
50
|
+
def test_a_call_target_past_every_row_is_outside_and_names_its_near_calling_site(self):
|
|
51
|
+
r = self.check("start\tsize\n1000:0000\t6\n1000:0006\t1\n", controls=[0, 3])
|
|
52
|
+
self.assertEqual(r["targets"], [{
|
|
53
|
+
"target": 0x0B, "address": "1000:000B", "status": "outside every row", "evidence": "entry-path call",
|
|
54
|
+
"site": 3, "call": "near", "siteClassification": "entry-path instruction", "region": "resident",
|
|
55
|
+
"provenance": r["targets"][0]["provenance"], "callSites": {"entryPath": 1, "contested": 0, "rawCandidates": 0}}])
|
|
56
|
+
self.assertEqual(r["targets"][0]["provenance"]["encoding"], "relative16")
|
|
57
|
+
path = r["counts"]["entry-path call"]
|
|
58
|
+
self.assertEqual((path["targets"], path["inventoryStarts"], path["outsideEveryRow"], path["insideAnotherRow"]), (2, 1, 1, 0))
|
|
59
|
+
self.assertEqual(r["counts"]["callTargets"], 2)
|
|
60
|
+
self.assertTrue(r["summary"].startswith("1 of the 2 call targets that entry-path calls resolve to are not inventory "
|
|
61
|
+
"starts: 0 inside another row's body, 1 outside every row."))
|
|
62
|
+
self.assertEqual([c["site"] for c in r["controls"]], [0, 3])
|
|
63
|
+
self.assertFalse(r["partialSearch"])
|
|
64
|
+
self.assertFalse(r["truncated"])
|
|
65
|
+
|
|
66
|
+
def test_a_target_inside_another_rows_body_names_that_row_and_an_alias_spelling_is_a_start(self):
|
|
67
|
+
# 0FFF:0016 is the byte 1000:0006 names, so 0006 is a start and 000B lies in that row's six bytes.
|
|
68
|
+
r = self.check("start\tsize\tname\n1000:0000\t6\tfirst\n0FFF:0016\t6\tsecond\n")
|
|
69
|
+
self.assertEqual([(t["target"], t["status"], t["rows"]) for t in r["targets"]],
|
|
70
|
+
[(0x0B, "inside another row's body", [{"start": "0FFF:0016", "size": 6, "name": "second"}])])
|
|
71
|
+
self.assertEqual(r["counts"]["entry-path call"]["insideAnotherRow"], 1)
|
|
72
|
+
|
|
73
|
+
def test_a_long_row_before_the_target_still_names_every_row_whose_body_holds_it(self):
|
|
74
|
+
# The long row 0FFF:0010 covers 0000..000F; the short rows after it start past 000B or end before it.
|
|
75
|
+
r = self.check("start\tsize\tname\n0FFF:0010\t16\tlong\n1000:0001\t2\tshort\n1000:0006\t1\tret\n1000:0009\t3\tholder\n")
|
|
76
|
+
self.assertEqual([(t["target"], [row["name"] for row in t["rows"]]) for t in r["targets"]],
|
|
77
|
+
[(0x0B, ["long", "holder"])])
|
|
78
|
+
|
|
79
|
+
def test_a_ranges_column_gives_the_body_and_its_ends_are_exclusive(self):
|
|
80
|
+
inside = self.check("start\tsize\tranges\n1000:0000\t6\t\n1000:0006\t2\t1000:0006..1000:0007 1000:000B..1000:000C\n")
|
|
81
|
+
self.assertEqual([(t["target"], t["status"]) for t in inside["targets"]], [(0x0B, "inside another row's body")])
|
|
82
|
+
past = self.check("start\tsize\tranges\n1000:0000\t6\t\n1000:0006\t2\t1000:0006..1000:0007 1000:000A..1000:000B\n")
|
|
83
|
+
self.assertEqual([(t["target"], t["status"]) for t in past["targets"]], [(0x0B, "outside every row")])
|
|
84
|
+
|
|
85
|
+
def test_an_overlay_far_call_through_a_trampoline_is_placed_by_file_offset(self):
|
|
86
|
+
# 0000 calls far through an FBOV fixup whose trampoline names the overlay entry at file offset 0010.
|
|
87
|
+
data = bytes.fromhex("9a00000800" "c3") + bytes(10) + bytes.fromhex("c3") + bytes(7)
|
|
88
|
+
regions = [{"name": "resident", "start": 0, "end": 6, "ip": 0, "segment": 0x1000, "resident": True,
|
|
89
|
+
"entries": [0], "evidence": "synthetic resident code"},
|
|
90
|
+
{"name": "overlay", "start": 0x10, "end": 0x18, "ip": 0, "segment": 0x2000, "entries": [0x10],
|
|
91
|
+
"container": {"view": "overlay 1", "start": 0x10, "end": 0x18}, "evidence": "synthetic overlay view"}]
|
|
92
|
+
fixup = {"site": 3, "raw": 8, "descriptor": 1, "segment": 0x2000, "target": 0x10, "trampoline": 0x40,
|
|
93
|
+
"evidence": "synthetic FBOV descriptor/fixup"}
|
|
94
|
+
run = lambda text: run_report(data, config(data, regions, relocations=[fixup], inventory=self.inventory(text)),
|
|
95
|
+
"inventory-check")
|
|
96
|
+
r = run("start\tsize\n1000:0000\t6\n")
|
|
97
|
+
self.assertEqual([(t["target"], t["address"], t["status"], t["call"], t["site"]) for t in r["targets"]],
|
|
98
|
+
[(0x10, "0x10", "outside every row", "far", 0)])
|
|
99
|
+
self.assertEqual(r["targets"][0]["provenance"]["relocation"]["trampoline"], 0x40)
|
|
100
|
+
self.assertEqual(run("start\tsize\n1000:0000\t6\n0x10\t1\n")["targets"], [])
|
|
101
|
+
# A row that writes the overlay by its analysis segment lies in no declared code, and says so.
|
|
102
|
+
aliased = run("start\tsize\n1000:0000\t6\n2000:0000\t1\n")
|
|
103
|
+
self.assertEqual(aliased["rowsOutsideDeclaredCode"], ["2000:0000"])
|
|
104
|
+
self.assertEqual(aliased["counts"]["rowsOutsideDeclaredCode"], 1)
|
|
105
|
+
self.assertEqual(aliased["targets"][0]["status"], "outside every row")
|
|
106
|
+
|
|
107
|
+
def test_a_target_only_raw_bytes_call_is_kept_apart_from_entry_path_targets(self):
|
|
108
|
+
r = self.check("start\tsize\n1000:0000\t6\n1000:0006\t1\n1000:000B\t1\n", data=RAW)
|
|
109
|
+
self.assertEqual([(t["target"], t["evidence"], t["siteClassification"]) for t in r["targets"]],
|
|
110
|
+
[(0x0F, "raw byte candidate only", "raw byte candidate")])
|
|
111
|
+
self.assertEqual(r["counts"]["entry-path call"]["outsideEveryRow"], 0)
|
|
112
|
+
self.assertEqual(r["counts"]["raw byte candidate only"]["outsideEveryRow"], 1)
|
|
113
|
+
self.assertIn("0 of the 2 call targets", r["summary"])
|
|
114
|
+
self.assertIn("1 more comes only from contested instructions or raw byte candidates", r["summary"])
|
|
115
|
+
|
|
116
|
+
def test_an_unresolved_call_is_listed_and_named_in_the_summary(self):
|
|
117
|
+
data = bytes.fromhex("ffd3" "c3") # call bx; ret
|
|
118
|
+
r = self.check("start\tsize\n1000:0000\t3\n", data=data)
|
|
119
|
+
self.assertEqual([(u["site"], u["target"]) for u in r["unresolved"]], [(0, None)])
|
|
120
|
+
self.assertEqual(r["counts"]["unresolvedCalls"], 1)
|
|
121
|
+
self.assertIn("1 call in the searched regions is unresolved, 1 of them on the entry path", r["summary"])
|
|
122
|
+
|
|
123
|
+
def test_the_summary_says_how_many_unresolved_calls_are_raw_byte_candidates(self):
|
|
124
|
+
# 000C is an unreached far call with no fixup on its segment word.
|
|
125
|
+
r = self.check("start\tsize\n1000:0000\t6\n1000:0006\t1\n1000:000B\t1\n", data=CALLS + bytes.fromhex("9a00000000" "c3"))
|
|
126
|
+
self.assertEqual([(u["site"], u["classification"]) for u in r["unresolved"]], [(12, "raw byte candidate")])
|
|
127
|
+
self.assertIn("1 call in the searched regions is unresolved, 0 of them on the entry path", r["summary"])
|
|
128
|
+
|
|
129
|
+
def test_a_target_outside_declared_code_is_left_out_of_the_compared_total(self):
|
|
130
|
+
# The FBOV fixup names file offset 0010, which no declared region holds.
|
|
131
|
+
data = bytes.fromhex("9a00000800" "c3") + bytes(18)
|
|
132
|
+
regions = [{"name": "resident", "start": 0, "end": 6, "ip": 0, "segment": 0x1000, "resident": True,
|
|
133
|
+
"entries": [0], "evidence": "synthetic resident code"}]
|
|
134
|
+
fixup = {"site": 3, "raw": 8, "descriptor": 1, "segment": 0x2000, "target": 0x10, "trampoline": 0x40,
|
|
135
|
+
"evidence": "synthetic FBOV descriptor/fixup"}
|
|
136
|
+
r = run_report(data, config(data, regions, relocations=[fixup], inventory=self.inventory("start\tsize\n1000:0000\t6\n")),
|
|
137
|
+
"inventory-check")
|
|
138
|
+
self.assertEqual([(t["target"], t["address"], t["status"]) for t in r["targets"]], [(0x10, None, "outside declared code")])
|
|
139
|
+
self.assertEqual(r["counts"]["entry-path call"]["outsideDeclaredCode"], 1)
|
|
140
|
+
self.assertTrue(r["summary"].startswith("0 of the 0 call targets that entry-path calls resolve to are not inventory "
|
|
141
|
+
"starts: 0 inside another row's body, 0 outside every row. 1 more entry-path "
|
|
142
|
+
"call target lies outside declared code and is not compared with the inventory."))
|
|
143
|
+
|
|
144
|
+
def test_a_scan_limit_short_of_the_code_makes_the_search_partial(self):
|
|
145
|
+
# The region lies in no container or declared segment, so only the scan gap shows the stop.
|
|
146
|
+
r = self.check("start\tsize\n1000:0000\t6\n1000:0006\t1\n", data=RAW, scanLimit=12)
|
|
147
|
+
self.assertEqual([(g["unsearchedStart"], g["reason"]) for g in r["gaps"] if g.get("reason") == "raw scan limit"],
|
|
148
|
+
[(12, "raw scan limit")])
|
|
149
|
+
self.assertIn("The search is partial.", r["summary"])
|
|
150
|
+
self.assertTrue(r["partialSearch"])
|
|
151
|
+
self.assertEqual([c["unsearched"] for c in r["coverage"]], [[{"start": 12, "end": len(RAW)}]])
|
|
152
|
+
# incoming reads the same coverage, so its search is partial too.
|
|
153
|
+
incoming = run_report(RAW, config(RAW, target=0x0B, scanLimit=12), "incoming")
|
|
154
|
+
self.assertTrue(incoming["partialSearch"])
|
|
155
|
+
self.assertFalse(run_report(RAW, config(RAW, target=0x0B), "incoming")["partialSearch"])
|
|
156
|
+
walked = self.check("start\tsize\n1000:0000\t6\n1000:0006\t1\n", instructionLimit=1)
|
|
157
|
+
self.assertIn("The entry-path walk stopped at its instruction limit", walked["summary"])
|
|
158
|
+
|
|
159
|
+
def test_an_inventory_past_its_row_or_size_limit_is_refused(self):
|
|
160
|
+
from scientific_method_engine.x86 import inventory
|
|
161
|
+
with self.assertRaisesRegex(ValueError, f"more than {inventory.MAX_ROWS} rows"):
|
|
162
|
+
self.check("start\tsize\n" + "1000:0000\t1\n" * (inventory.MAX_ROWS + 1))
|
|
163
|
+
with self.assertRaisesRegex(ValueError, "at most 16 MiB"):
|
|
164
|
+
self.check("start\tsize\n" + " " * inventory.MAX_INVENTORY)
|
|
165
|
+
|
|
166
|
+
def test_flat32_code_is_placed_by_its_virtual_address(self):
|
|
167
|
+
data = bytes.fromhex("e800000000" "c3") # call 00401005; ret
|
|
168
|
+
regions = [{"name": "text", "start": 0, "end": len(data), "ip": 0x401000, "segment": 0,
|
|
169
|
+
"entries": [0], "evidence": "synthetic flat code"}]
|
|
170
|
+
r = run_report(data, config(data, regions, sourceKind="synthetic-raw", addressModel="flat32", bits=32,
|
|
171
|
+
inventory=self.inventory("start\tsize\n0x00401000\t5\n")), "inventory-check")
|
|
172
|
+
self.assertEqual([(t["target"], t["address"], t["status"]) for t in r["targets"]],
|
|
173
|
+
[(5, "0x00401005", "outside every row")])
|
|
174
|
+
|
|
175
|
+
def test_the_result_limit_truncates_and_says_so(self):
|
|
176
|
+
r = self.check("start\tsize\n1000:0000\t1\n", limit=1)
|
|
177
|
+
self.assertEqual(len(r["targets"]), 1)
|
|
178
|
+
self.assertTrue(r["truncated"])
|
|
179
|
+
self.assertEqual(r["counts"]["entry-path call"]["outsideEveryRow"], 2)
|
|
180
|
+
|
|
181
|
+
def test_a_missed_control_and_a_malformed_inventory_are_refused(self):
|
|
182
|
+
with self.assertRaisesRegex(ValueError, "Positive control 12 missed"):
|
|
183
|
+
self.check("start\tsize\n1000:0000\t6\n", data=RAW, controls=[12])
|
|
184
|
+
for text, message in (("start\tname\n1000:0000\tmain\n", "columns are start and size"),
|
|
185
|
+
("start\tsize\n1000:0000\t0\n", "line 2: size 0"),
|
|
186
|
+
("start\tsize\n1000:0000\t6\n1000:0000\t2\n", "line 3: start 1000:0000 is listed twice"),
|
|
187
|
+
("start\tsize\nmain\t6\n", "line 2: start main is not an address"),
|
|
188
|
+
("start\tsize\tranges\n1000:0000\t5\t1000:0000..1000:0006\n", "line 2: size 5 is not the total"),
|
|
189
|
+
("start\tsize\n1000:0000\n", "line 2: has 1 cells")):
|
|
190
|
+
with self.subTest(text=text):
|
|
191
|
+
with self.assertRaisesRegex(ValueError, message):
|
|
192
|
+
self.check(text)
|
|
193
|
+
with self.assertRaisesRegex(ValueError, "inventory must be an absolute path"):
|
|
194
|
+
run_report(CALLS, config(inventory="inventory.tsv"), "inventory-check")
|
|
195
|
+
with self.assertRaisesRegex(ValueError, "needs inventory"):
|
|
196
|
+
run_report(CALLS, config(), "inventory-check")
|
|
197
|
+
with self.assertRaisesRegex(ValueError, "inventory applies only to inventory-check"):
|
|
198
|
+
run_report(CALLS, config(target=0x0B, inventory=self.inventory("start\tsize\n")), "incoming")
|
|
199
|
+
|
|
200
|
+
def test_the_cli_reads_inventory_beside_a_config_file_and_refuses_a_relative_one_from_the_reader(self):
|
|
201
|
+
(self.root / "fixture.bin").write_bytes(CALLS)
|
|
202
|
+
self.inventory("start\tsize\r\n1000:0000\t6\r\n1000:0006\t1\r\n")
|
|
203
|
+
cfg = config(source="fixture.bin", xxh3=xxhash.xxh3_128_hexdigest(CALLS), inventory="inventory.tsv")
|
|
204
|
+
path = self.root / "config.json"
|
|
205
|
+
path.write_text(json.dumps(cfg))
|
|
206
|
+
result = subprocess.run([*ENGINE, "inventory-check", str(path)], capture_output=True, text=True, env=ENGINE_ENV)
|
|
207
|
+
self.assertEqual(result.returncode, 0, result.stderr)
|
|
208
|
+
report = json.loads(result.stdout)
|
|
209
|
+
self.assertEqual([t["target"] for t in report["targets"]], [0x0B])
|
|
210
|
+
self.assertEqual(report["inventory"], {"path": str(self.root / "inventory.tsv"), "rows": 2})
|
|
211
|
+
prepared = {**cfg, "source": str(self.root / "fixture.bin"), "preparedProtocol": 3}
|
|
212
|
+
piped = subprocess.run([*ENGINE, "inventory-check", "-"], input=json.dumps(prepared), capture_output=True,
|
|
213
|
+
text=True, env=ENGINE_ENV)
|
|
214
|
+
self.assertEqual(piped.returncode, 1)
|
|
215
|
+
self.assertIn("relative inventory path", piped.stderr)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
if __name__ == "__main__":
|
|
219
|
+
unittest.main()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_effect_order.py
RENAMED
|
File without changes
|
{scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_enter_frames.py
RENAMED
|
File without changes
|
{scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_entry_frame.py
RENAMED
|
File without changes
|
|
File without changes
|
{scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_interrupt_models.py
RENAMED
|
File without changes
|
|
File without changes
|
{scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_memory_scopes.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_return_checks.py
RENAMED
|
File without changes
|
{scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_successor_rule.py
RENAMED
|
File without changes
|
|
File without changes
|
{scientific_method_engine-13.4.0 → scientific_method_engine-13.5.0}/tests/test_term_limit.py
RENAMED
|
File without changes
|
|
File without changes
|