scientific-method-engine 19.8.0__tar.gz → 19.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/PKG-INFO +1 -1
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/pyproject.toml +1 -1
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/cli.py +1 -1
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/inventory.py +6 -3
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/reach.py +12 -6
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/reports.py +3 -0
- scientific_method_engine-19.9.0/src/scientific_method_engine/x86/scan_index.py +181 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/trace.py +28 -7
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_inventory_check.py +23 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_reach.py +80 -5
- scientific_method_engine-19.9.0/tests/test_scan_index.py +163 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/.gitignore +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/LICENSE +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/README.md +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/__init__.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/__main__.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/__init__.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/derived_counts.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/image.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/loops.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/machine.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/pcode.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/pe.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/ranges.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/relational.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/table_bounds.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/src/scientific_method_engine/x86/values.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/oracle.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_config_encoding.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_continuation_budget.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_derived_counts.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_dispatch.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_effect_order.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_enter_frames.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_entry_frame.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_indirect_far_transfers.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_interrupt_models.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_join_terms.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_loops.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_memory_scopes.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_nested_frame_request.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_oracle.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_pe.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_ranges.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_relational_controls.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_return_checks.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_successor_rule.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_table_bounds.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_table_continuations.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_term_limit.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_upstream_gaps.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_volatile_memory.py +0 -0
- {scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_x86.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: scientific-method-engine
|
|
3
|
-
Version: 19.
|
|
3
|
+
Version: 19.9.0
|
|
4
4
|
Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
|
|
5
5
|
Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
|
|
6
6
|
Author: kibertoad
|
|
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "scientific-method-engine"
|
|
7
7
|
# The release workflow writes the published version from the package's release tag.
|
|
8
|
-
version = "19.
|
|
8
|
+
version = "19.9.0"
|
|
9
9
|
description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
|
|
10
10
|
readme = "README.md"
|
|
11
11
|
requires-python = ">=3.12"
|
|
@@ -13,7 +13,7 @@ PREPARED_CONFIG_LIMIT = 16 * 1024 * 1024
|
|
|
13
13
|
# The header names the decoder and instruction semantics that actually ran, not the pins in pyproject.toml.
|
|
14
14
|
DECODER = f"capstone {CAPSTONE_VERSION or 'without distribution metadata'}"
|
|
15
15
|
INSTRUCTION_SEMANTICS = f"pypcode {pypcode.__version__} (Ghidra SLEIGH x86)"
|
|
16
|
-
USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bounds|owner|callees|reach|trace|uses|arguments|"
|
|
16
|
+
USAGE = ("Usage: scientific-method-engine <scan-index|operand|operand-candidates|target|bounds|owner|callees|reach|trace|uses|arguments|"
|
|
17
17
|
"effects|returns|memory|incoming|inventory-check|call-order|guards|allocation|dispatch> <config.json|->\n"
|
|
18
18
|
"effects includes ordered path writes/calls and local restoration witnesses; transactionality remains unestablished.\n"
|
|
19
19
|
"callees compares its edges with an ExportCallEdges.java export given as ghidraCallEdges;\n"
|
|
@@ -239,15 +239,18 @@ def _no_return_check(image, routines, declared, limit):
|
|
|
239
239
|
``routines`` are the declared routines, or the targets of the declared calls. The walk starts at
|
|
240
240
|
them and continues past every interrupt except a declared one, as ``reach`` reads them, so a
|
|
241
241
|
routine that ends in an interrupt that returns is contradicted by what follows it. Calls are
|
|
242
|
-
stepped over at their return sites,
|
|
242
|
+
stepped over at their return sites with their targets resolved, so a call to a declared routine
|
|
243
|
+
or at a declared site still ends its branch, and no callee other than a declared routine is
|
|
244
|
+
decoded or spends ``limit``: the walk reads only the routines' and the declared routines' own paths.
|
|
243
245
|
A routine the walk did not establish, one that does not decode or that a rejected overlap removed,
|
|
244
246
|
has no return sites and is not in ``read``.
|
|
245
247
|
"""
|
|
246
248
|
from .reach import returns_from, successor_graph
|
|
247
249
|
if not routines:
|
|
248
250
|
return {}, {}, False
|
|
249
|
-
seen, gaps, _, _, _ = walk(image, sorted(routines), limit, follow_interrupts=True,
|
|
250
|
-
|
|
251
|
+
seen, gaps, _, _, _ = walk(image, sorted(routines), limit, follow_interrupts=True, resolve_over_calls=True,
|
|
252
|
+
**declared)
|
|
253
|
+
graph = successor_graph(image, seen, follow_interrupts=True, resolve_over_calls=True, **declared)
|
|
251
254
|
return ({at: returns_from(graph, seen, set(), at) for at in routines}, seen,
|
|
252
255
|
any(g["reason"] == LIMIT_REASON for g in gaps))
|
|
253
256
|
|
|
@@ -1046,10 +1046,13 @@ def reach(image, config):
|
|
|
1046
1046
|
or any(at in seen or at in contested or at in overlapping for at in stop_sites))
|
|
1047
1047
|
if stop_met and checked:
|
|
1048
1048
|
# A stop cuts the query and says nothing about the build, so the return check reads the reached
|
|
1049
|
-
# declared routines without the stops: code a stop hid can still contradict a declaration.
|
|
1049
|
+
# declared routines without the stops: code a stop hid can still contradict a declaration. It
|
|
1050
|
+
# steps over calls with their targets resolved, entering only a declared routine, so only the
|
|
1051
|
+
# declared routines' own paths are decoded and spend the instruction limit; returns_from skips
|
|
1052
|
+
# call edges either way.
|
|
1050
1053
|
read, check_gaps, _, _, check_contested = walk(image, sorted(at for at in checked if at in distance),
|
|
1051
1054
|
instruction_limit, follow_interrupts=True,
|
|
1052
|
-
stops=frozenset(leaves), **declared)
|
|
1055
|
+
stops=frozenset(leaves), resolve_over_calls=True, **declared)
|
|
1053
1056
|
check_stopped = any(g["reason"] == LIMIT_REASON for g in check_gaps)
|
|
1054
1057
|
# What the check could not read past a stop leaves its declarations unchecked there, as a gap of the
|
|
1055
1058
|
# query's own walk would; the rows the query's walk already holds are not repeated.
|
|
@@ -1059,7 +1062,8 @@ def reach(image, config):
|
|
|
1059
1062
|
and g["site"] not in check_contested]
|
|
1060
1063
|
+ [{"site": at, "reason": CHECK_CONTESTED} for at in check_contested if at not in contested],
|
|
1061
1064
|
key=lambda g: (g["site"], g["reason"]))
|
|
1062
|
-
check_graph = successor_graph(image, read, kept=frozenset(leaves), follow_interrupts=True,
|
|
1065
|
+
check_graph = successor_graph(image, read, kept=frozenset(leaves), follow_interrupts=True,
|
|
1066
|
+
resolve_over_calls=True, **declared)
|
|
1063
1067
|
returns = {at: returns_from(check_graph, read, returning_leaves, at) for at in checked}
|
|
1064
1068
|
no_return = no_return_rows(no_return_routines, no_return_interrupts, returns, ended_sites, seen, distance, read,
|
|
1065
1069
|
no_return_sites, site_targets)
|
|
@@ -1157,9 +1161,11 @@ def reach(image, config):
|
|
|
1157
1161
|
"A stop narrows the question and claims nothing about the build: the walk reads no "
|
|
1158
1162
|
"further at a stop site, nor into a stop routine or past a call to it, so a target "
|
|
1159
1163
|
"reached only past a stop is not reached, and negativeUsable answers the narrowed "
|
|
1160
|
-
"question. The noReturn return check reads past the stops, with a walk of its own
|
|
1161
|
-
"
|
|
1162
|
-
"
|
|
1164
|
+
"question. The noReturn return check reads past the stops, with a walk of its own that "
|
|
1165
|
+
"steps over calls at their return sites after resolving their targets, so it decodes "
|
|
1166
|
+
"only the declared routines' own paths; returnCheckGaps lists what that walk could not "
|
|
1167
|
+
"read on those paths that the query's walk did not record, and any row keeps "
|
|
1168
|
+
"negativeUsable false, as a gap would. "
|
|
1163
1169
|
"decodedRanges, when asked for, covers the instructions the walk decoded; no range "
|
|
1164
1170
|
"starts at a leaf, a stop routine or a contested, unresolved overlapping or undecodable start, though "
|
|
1165
1171
|
"one can lie inside a decoded instruction. A site inside a range lies in a decoded "
|
|
@@ -1884,6 +1884,9 @@ def _run_report(image, config, command):
|
|
|
1884
1884
|
raise ValueError("indirectCalls applies only to reach and inventory-check")
|
|
1885
1885
|
if command == "operand":
|
|
1886
1886
|
return operand_provenance(image, config)
|
|
1887
|
+
if command == "scan-index":
|
|
1888
|
+
from .scan_index import scan_index
|
|
1889
|
+
return scan_index(image, config)
|
|
1887
1890
|
if command == "target":
|
|
1888
1891
|
return call_target_report(image, config)
|
|
1889
1892
|
if command == "bounds":
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""Reusable raw caller/operand candidates. No reachability or instruction-start claims."""
|
|
2
|
+
import hashlib
|
|
3
|
+
from contextlib import closing
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
import sqlite3
|
|
8
|
+
import tempfile
|
|
9
|
+
|
|
10
|
+
from capstone import CS_GRP_CALL
|
|
11
|
+
from capstone.x86 import X86_OP_IMM, X86_OP_MEM
|
|
12
|
+
import xxhash
|
|
13
|
+
|
|
14
|
+
from .image import CAPSTONE_VERSION, integer, scan_limit
|
|
15
|
+
from .reports import _relative_transfer
|
|
16
|
+
from .trace import call_target, unsupported_transfer
|
|
17
|
+
|
|
18
|
+
SCHEMA = 2
|
|
19
|
+
MAX_INDEX_BYTES = 256 * 1024 * 1024
|
|
20
|
+
INSERT_BATCH = 10000
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _identity(image):
|
|
24
|
+
# Query, output location and source pathname do not change the decoded corpus.
|
|
25
|
+
config = {k: v for k, v in image.config.items() if k not in ("query", "scanIndex", "source")}
|
|
26
|
+
# The digest keeps the stored metadata and every report small however large the prepared config is.
|
|
27
|
+
config_digest = hashlib.sha256(json.dumps(config, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
|
28
|
+
sources = hashlib.sha256()
|
|
29
|
+
root = Path(__file__).resolve().parent.parent
|
|
30
|
+
for path in sorted(root.rglob("*.py")):
|
|
31
|
+
sources.update(path.relative_to(root).as_posix().encode())
|
|
32
|
+
sources.update(path.read_bytes())
|
|
33
|
+
return {"schema": SCHEMA, "xxh3": xxhash.xxh3_128_hexdigest(image.data),
|
|
34
|
+
"size": len(image.data), "capstone": CAPSTONE_VERSION,
|
|
35
|
+
"engineSources": sources.hexdigest(), "configSha256": config_digest}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _build(image, destination, identity):
|
|
39
|
+
handle, temporary = tempfile.mkstemp(prefix="scan-index-", suffix=".partial", dir=destination.parent)
|
|
40
|
+
os.close(handle)
|
|
41
|
+
try:
|
|
42
|
+
with closing(sqlite3.connect(temporary)) as db, db:
|
|
43
|
+
# Bound disk use, including the final query index. No partial index is published.
|
|
44
|
+
db.execute("PRAGMA page_size=4096")
|
|
45
|
+
db.execute(f"PRAGMA max_page_count={MAX_INDEX_BYTES // 4096}")
|
|
46
|
+
db.execute("CREATE TABLE candidates (site INTEGER, size INTEGER, kind TEXT, value INTEGER)")
|
|
47
|
+
db.execute("CREATE TABLE metadata (value TEXT)")
|
|
48
|
+
db.execute("CREATE TABLE undecoded (start INTEGER, end INTEGER)")
|
|
49
|
+
db.execute("CREATE TABLE unresolved_calls (site INTEGER, size INTEGER)")
|
|
50
|
+
undecoded = unresolved = rows = 0
|
|
51
|
+
pending = []
|
|
52
|
+
for region in image.regions:
|
|
53
|
+
gap = None
|
|
54
|
+
for site in range(region["start"], region["end"]):
|
|
55
|
+
ins = image.decode(site)
|
|
56
|
+
# The one-shot decoder cache is unnecessary during a whole-corpus scan.
|
|
57
|
+
image.cache.pop(site, None)
|
|
58
|
+
if ins is None:
|
|
59
|
+
undecoded += 1
|
|
60
|
+
gap = site if gap is None else gap
|
|
61
|
+
continue
|
|
62
|
+
if gap is not None:
|
|
63
|
+
db.execute("INSERT INTO undecoded VALUES (?, ?)", (gap, site))
|
|
64
|
+
gap = None
|
|
65
|
+
found = set()
|
|
66
|
+
for operand in ins.operands:
|
|
67
|
+
# Only encoded literals, as operand-candidates reads them: implicit forms (SHL r,1;
|
|
68
|
+
# [BX]) and relative branch targets are not immediates or displacements.
|
|
69
|
+
if operand.type == X86_OP_IMM and ins.imm_size and not _relative_transfer(ins):
|
|
70
|
+
found.add(("immediate", operand.imm))
|
|
71
|
+
elif operand.type == X86_OP_MEM and ins.disp_size:
|
|
72
|
+
found.add(("displacement", operand.mem.disp))
|
|
73
|
+
if ins.group(CS_GRP_CALL):
|
|
74
|
+
# An operand-size-overridden call is outside the frame model, as direct_calls reports it.
|
|
75
|
+
target = None if unsupported_transfer(image, ins) else call_target(image, site, ins)[0]
|
|
76
|
+
if target is None:
|
|
77
|
+
unresolved += 1
|
|
78
|
+
db.execute("INSERT INTO unresolved_calls VALUES (?, ?)", (site, ins.size))
|
|
79
|
+
else:
|
|
80
|
+
found.add(("call", target))
|
|
81
|
+
pending.extend((site, ins.size, kind, value) for kind, value in sorted(found))
|
|
82
|
+
rows += len(found)
|
|
83
|
+
if len(pending) >= INSERT_BATCH:
|
|
84
|
+
db.executemany("INSERT INTO candidates VALUES (?, ?, ?, ?)", pending)
|
|
85
|
+
pending.clear()
|
|
86
|
+
if gap is not None:
|
|
87
|
+
db.execute("INSERT INTO undecoded VALUES (?, ?)", (gap, region["end"]))
|
|
88
|
+
db.executemany("INSERT INTO candidates VALUES (?, ?, ?, ?)", pending)
|
|
89
|
+
db.execute("CREATE INDEX lookup ON candidates (kind, value, site)")
|
|
90
|
+
metadata = {"identity": identity, "scannedBytes": image.code_bytes,
|
|
91
|
+
"undecodedOffsets": undecoded, "unresolvedCallCandidates": unresolved,
|
|
92
|
+
"candidateRows": rows}
|
|
93
|
+
db.execute("INSERT INTO metadata VALUES (?)", (json.dumps(metadata, sort_keys=True),))
|
|
94
|
+
# A concurrent builder of the same identity produces the same corpus. On Windows the replace
|
|
95
|
+
# fails while another run has that corpus open, which then already holds the same rows.
|
|
96
|
+
try:
|
|
97
|
+
os.replace(temporary, destination)
|
|
98
|
+
except PermissionError:
|
|
99
|
+
if not destination.exists():
|
|
100
|
+
raise
|
|
101
|
+
finally:
|
|
102
|
+
if os.path.exists(temporary):
|
|
103
|
+
os.unlink(temporary)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def scan_index(image, config):
|
|
107
|
+
"""Query a content-keyed local SQLite index of every-offset decode candidates.
|
|
108
|
+
|
|
109
|
+
``scanIndex.directory`` must be an absolute directory.
|
|
110
|
+
``query`` names ``kind`` (immediate, displacement, call), integer ``value``,
|
|
111
|
+
``limit`` (1..10000, default 1000) and optional matching-site ``controls``.
|
|
112
|
+
A call value is a mapped source-file offset, not a loaded address. Every
|
|
113
|
+
result remains a raw candidate, including prefix and interior decodes.
|
|
114
|
+
``undecodedRanges`` (half-open) and ``unresolvedCalls`` list where the scan
|
|
115
|
+
could not see, up to ``limit`` rows each.
|
|
116
|
+
Source verification and MZ/FBOV preparation stay with the normal reader.
|
|
117
|
+
"""
|
|
118
|
+
options, query = config.get("scanIndex"), config.get("query")
|
|
119
|
+
if not isinstance(options, dict) or set(options) != {"directory"}:
|
|
120
|
+
raise ValueError("scanIndex needs only directory")
|
|
121
|
+
directory = options["directory"]
|
|
122
|
+
if not isinstance(directory, str) or not Path(directory).is_absolute():
|
|
123
|
+
raise ValueError("scanIndex.directory must be an absolute path under GAME_DIR")
|
|
124
|
+
if not isinstance(query, dict) or set(query) - {"kind", "value", "limit", "controls"}:
|
|
125
|
+
raise ValueError("scan-index query needs kind, value, optional limit and controls")
|
|
126
|
+
kind = query.get("kind")
|
|
127
|
+
if kind not in ("immediate", "displacement", "call"):
|
|
128
|
+
raise ValueError("scan-index kind must be immediate, displacement or call")
|
|
129
|
+
value = integer(query.get("value"), -(1 << 63), (1 << 63) - 1, "query value")
|
|
130
|
+
limit = integer(query.get("limit", 1000), 1, 10000, "query limit")
|
|
131
|
+
controls = query.get("controls", [])
|
|
132
|
+
if not isinstance(controls, list) or len(controls) > 128:
|
|
133
|
+
raise ValueError("query controls must list at most 128 matching source offsets")
|
|
134
|
+
for site in controls:
|
|
135
|
+
integer(site, 0, len(image.data) - 1, "control site")
|
|
136
|
+
budget = scan_limit(image, config.get("scanLimit", 1048576))
|
|
137
|
+
if image.code_bytes > budget:
|
|
138
|
+
raise ValueError("scan-index scanLimit reached; no complete index produced")
|
|
139
|
+
identity = _identity(image)
|
|
140
|
+
key = hashlib.sha256(json.dumps(identity, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
|
|
141
|
+
folder = Path(directory)
|
|
142
|
+
folder.mkdir(parents=True, exist_ok=True)
|
|
143
|
+
destination = folder / (key + ".sqlite")
|
|
144
|
+
hit = destination.exists()
|
|
145
|
+
if not hit:
|
|
146
|
+
try:
|
|
147
|
+
_build(image, destination, identity)
|
|
148
|
+
except sqlite3.Error as error:
|
|
149
|
+
raise ValueError(f"scan-index build failed and published nothing: {error}") from None
|
|
150
|
+
try:
|
|
151
|
+
if destination.stat().st_size > MAX_INDEX_BYTES:
|
|
152
|
+
raise ValueError("scan-index exceeds the 256 MiB cache limit")
|
|
153
|
+
with closing(sqlite3.connect(destination.as_uri() + "?mode=ro", uri=True)) as db:
|
|
154
|
+
raw = db.execute("SELECT value FROM metadata").fetchall()
|
|
155
|
+
if len(raw) != 1:
|
|
156
|
+
raise ValueError("scan-index metadata is invalid; remove this local cache and rebuild")
|
|
157
|
+
metadata = json.loads(raw[0][0])
|
|
158
|
+
if not isinstance(metadata, dict) or metadata.get("identity") != identity:
|
|
159
|
+
raise ValueError("scan-index identity differs; remove this local cache and rebuild")
|
|
160
|
+
missing = [site for site in controls if db.execute(
|
|
161
|
+
"SELECT 1 FROM candidates WHERE kind=? AND value=? AND site=? LIMIT 1",
|
|
162
|
+
(kind, value, site)).fetchone() is None]
|
|
163
|
+
if missing:
|
|
164
|
+
raise ValueError(f"scan-index matching-site controls missed: {missing}")
|
|
165
|
+
total = db.execute("SELECT count(*) FROM candidates WHERE kind=? AND value=?", (kind, value)).fetchone()[0]
|
|
166
|
+
rows = db.execute("SELECT site, size FROM candidates WHERE kind=? AND value=? ORDER BY site LIMIT ?",
|
|
167
|
+
(kind, value, limit)).fetchall()
|
|
168
|
+
# A negative result is only as complete as the offsets that decoded and the calls that resolved.
|
|
169
|
+
gaps = db.execute("SELECT start, end FROM undecoded ORDER BY start LIMIT ?", (limit + 1,)).fetchall()
|
|
170
|
+
calls = db.execute("SELECT site, size FROM unresolved_calls ORDER BY site LIMIT ?", (limit + 1,)).fetchall()
|
|
171
|
+
except (sqlite3.Error, json.JSONDecodeError) as error:
|
|
172
|
+
raise ValueError(f"scan-index cache failed; remove this local cache and rebuild: {error}") from None
|
|
173
|
+
return {"command": "scan-index", "cacheKey": key, "cacheHit": hit,
|
|
174
|
+
"scan": metadata, "query": query, "totalCandidates": total,
|
|
175
|
+
"resultLimitReached": total > limit,
|
|
176
|
+
"rawCandidates": [{"site": site, "size": size} for site, size in rows],
|
|
177
|
+
"undecodedRanges": [{"start": start, "end": end} for start, end in gaps[:limit]],
|
|
178
|
+
"undecodedRangesLimitReached": len(gaps) > limit,
|
|
179
|
+
"unresolvedCalls": [{"site": site, "size": size} for site, size in calls[:limit]],
|
|
180
|
+
"unresolvedCallsLimitReached": len(calls) > limit,
|
|
181
|
+
"meaning": "Every-offset decode candidates only; no entry-path qualification or absence proof."}
|
|
@@ -181,7 +181,7 @@ class Step(NamedTuple):
|
|
|
181
181
|
|
|
182
182
|
def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, follow_interrupts=False,
|
|
183
183
|
modeled_interrupts=frozenset(), no_return_calls=frozenset(), no_return_interrupts=frozenset(),
|
|
184
|
-
indirect_calls=None, no_return_call_sites=frozenset()):
|
|
184
|
+
indirect_calls=None, no_return_call_sites=frozenset(), resolve_over_calls=False):
|
|
185
185
|
"""The successors of ``ins`` decoded at ``at``, under the rule ``walk`` and the ``uses`` caller continuation share.
|
|
186
186
|
|
|
187
187
|
An unsupported transfer encoding, a return, an unconditional jump without a resolved target,
|
|
@@ -190,7 +190,12 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
|
|
|
190
190
|
conditional jump or loop continues at its resolved target and at the next instruction. A declared
|
|
191
191
|
indirect jump continues at its table rows only. A call continues at its resolved target and at
|
|
192
192
|
its return site; with ``step_over_calls`` it continues at its return site only and its target
|
|
193
|
-
is neither resolved nor reported.
|
|
193
|
+
is neither resolved nor reported. With ``resolve_over_calls`` it also continues at its return
|
|
194
|
+
site only, but the target is resolved first, so the declarations below still end their branches:
|
|
195
|
+
a call to a routine in ``no_return_calls`` ends the branch, an unresolved target still records
|
|
196
|
+
its gap, and a declared indirect call keeps its declaration's rule. Only a target in
|
|
197
|
+
``no_return_calls`` is entered and reported as an edge, since a declared routine's own paths are
|
|
198
|
+
ones the walk reads; no other callee is. A call whose resolved target is in ``no_return_calls``, and an
|
|
194
199
|
interrupt whose site is in ``no_return_interrupts``, have no return site: the query declared that
|
|
195
200
|
they never come back. A call whose site is in ``no_return_call_sites`` has no return site either,
|
|
196
201
|
whatever its target: the query declared that this one call never comes back. A call whose site
|
|
@@ -215,7 +220,9 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
|
|
|
215
220
|
return Step(targets, False, None, gaps, edges, supplied=True)
|
|
216
221
|
declaration = (indirect_calls or {}).get(at)
|
|
217
222
|
if declaration is not None:
|
|
218
|
-
|
|
223
|
+
# Under resolve_over_calls only a declared noReturn routine is entered, and successor_kinds counts the
|
|
224
|
+
# target edges as the call successors, so a target that is not entered gets no edge either.
|
|
225
|
+
targets = [t for t in declaration["targets"] if not resolve_over_calls or t in no_return_calls]
|
|
219
226
|
edges = [{"site": at, "target": target, "kind": m,
|
|
220
227
|
"provenance": {"encoding": "declared indirect call", "evidence": declaration["evidence"]}}
|
|
221
228
|
for target in targets]
|
|
@@ -232,7 +239,17 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
|
|
|
232
239
|
return Step([], True, None, [], [])
|
|
233
240
|
successors, gaps, edges = [], [], []
|
|
234
241
|
calls = m in ("call", "lcall")
|
|
235
|
-
if
|
|
242
|
+
if calls and resolve_over_calls:
|
|
243
|
+
# The target is resolved to apply the declarations. A declared noReturn routine is entered, since its own
|
|
244
|
+
# paths are ones the check reads; any other callee is neither entered nor reported as an edge.
|
|
245
|
+
target, provenance = call_target(image, at, ins)
|
|
246
|
+
if target is None:
|
|
247
|
+
edges.append({"site": at, "target": None, "kind": m, "provenance": provenance})
|
|
248
|
+
gaps.append({"site": at, "reason": provenance.get("reason", "target outside declared regions")})
|
|
249
|
+
elif target in no_return_calls:
|
|
250
|
+
edges.append({"site": at, "target": target, "kind": m, "provenance": provenance})
|
|
251
|
+
return Step([target], False, None, gaps, edges)
|
|
252
|
+
elif (calls and not step_over_calls) or m == "ljmp" or m.startswith(("j", "loop")):
|
|
236
253
|
target, provenance = call_target(image, at, ins)
|
|
237
254
|
edges.append({"site": at, "target": target, "kind": m, "provenance": provenance})
|
|
238
255
|
if target is None:
|
|
@@ -261,7 +278,7 @@ def cfg_step(image, at, ins, follow_flat_ports=False, step_over_calls=False, fol
|
|
|
261
278
|
|
|
262
279
|
def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts=False, stops=frozenset(),
|
|
263
280
|
modeled_interrupts=frozenset(), no_return_calls=frozenset(), no_return_interrupts=frozenset(),
|
|
264
|
-
indirect_calls=None, no_return_call_sites=frozenset(), ends=frozenset()):
|
|
281
|
+
indirect_calls=None, no_return_call_sites=frozenset(), ends=frozenset(), resolve_over_calls=False):
|
|
265
282
|
"""Decode the CFG reached from ``entries`` and check its instruction boundaries.
|
|
266
283
|
|
|
267
284
|
A port access in the flat model records a gap and ends that branch, as ``trace`` stops there.
|
|
@@ -274,7 +291,11 @@ def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts
|
|
|
274
291
|
decoded and its boundary checked, and the walk follows none of its successors.
|
|
275
292
|
``no_return_calls`` and ``no_return_interrupts`` end the branch at a call to one of those routines and at one of those
|
|
276
293
|
interrupt sites, ``no_return_call_sites`` at a call at one of those sites, and ``indirect_calls``
|
|
277
|
-
continues a declared indirect call at its declared targets, as ``cfg_step`` describes.
|
|
294
|
+
continues a declared indirect call at its declared targets, as ``cfg_step`` describes.
|
|
295
|
+
``resolve_over_calls`` steps over every call at its return site after resolving its target, so
|
|
296
|
+
those declarations still end their branches but no callee is entered other than a routine in
|
|
297
|
+
``no_return_calls``: the walk reads only the entries' and the declared routines' own paths, and
|
|
298
|
+
only they spend ``limit``. A
|
|
278
299
|
declared table whose count was derived from the code, and that the walk reads, gets a gap
|
|
279
300
|
(``derived_counts.entry_gaps``) when an edge the walk read enters the routine the count was
|
|
280
301
|
derived over other than at its start.
|
|
@@ -305,7 +326,7 @@ def walk(image, entries, limit=10000, follow_flat_ports=False, follow_interrupts
|
|
|
305
326
|
step = cfg_step(image, at, ins, follow_flat_ports, follow_interrupts=follow_interrupts,
|
|
306
327
|
modeled_interrupts=modeled_interrupts, no_return_calls=no_return_calls,
|
|
307
328
|
no_return_interrupts=no_return_interrupts, indirect_calls=indirect_calls,
|
|
308
|
-
no_return_call_sites=no_return_call_sites)
|
|
329
|
+
no_return_call_sites=no_return_call_sites, resolve_over_calls=resolve_over_calls)
|
|
309
330
|
successors[at] = step.successors
|
|
310
331
|
gaps.extend(step.gaps)
|
|
311
332
|
edges.extend(step.edges)
|
{scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_inventory_check.py
RENAMED
|
@@ -431,6 +431,29 @@ class InventoryCheckTests(unittest.TestCase):
|
|
|
431
431
|
self.assertIn("The walk that checks the noReturn routines stopped at its instruction limit", stopped["summary"])
|
|
432
432
|
self.assertFalse(r["noReturnCheckLimitReached"])
|
|
433
433
|
|
|
434
|
+
def test_the_no_return_check_reads_only_the_routines_own_paths(self):
|
|
435
|
+
# 0000 ret is the whole entry path. The declared routine at 0001 calls 0008, whose seven
|
|
436
|
+
# instructions a check that entered them would spend a limit of four on; its own path holds two.
|
|
437
|
+
data = bytes.fromhex("c3" "e80400" "ebfe" "9090" + "90" * 6 + "c3")
|
|
438
|
+
inventory = "start\tsize\n1000:0000\t1\n1000:0001\t7\n1000:0008\t7\n"
|
|
439
|
+
exit_routine = [{"routine": 1, "reason": "synthetic exit"}]
|
|
440
|
+
r = self.check(inventory, data=data, noReturn=exit_routine, instructionLimit=4)
|
|
441
|
+
self.assertFalse(r["noReturnCheckLimitReached"])
|
|
442
|
+
row = r["noReturn"][0]
|
|
443
|
+
self.assertEqual((row["returnSites"], row["contradicted"], row["read"], row["reached"]), ([], False, True, False))
|
|
444
|
+
# A return on the routine's own path past the stepped-over call still contradicts the declaration.
|
|
445
|
+
returning = bytes.fromhex("c3" "e80400" "c3" "909090" + "90" * 6 + "c3")
|
|
446
|
+
r = self.check(inventory, data=returning, noReturn=exit_routine, instructionLimit=4)
|
|
447
|
+
self.assertFalse(r["noReturnCheckLimitReached"])
|
|
448
|
+
self.assertEqual((r["noReturn"][0]["returnSites"], r["noReturn"][0]["contradicted"]), ([4], True))
|
|
449
|
+
# A declared indirect call on the routine's own path is stepped over too, and the return past it
|
|
450
|
+
# still contradicts the declaration.
|
|
451
|
+
declared_call = bytes.fromhex("c3" "ffd3" "c3" "c3")
|
|
452
|
+
r = self.check("start\tsize\n1000:0000\t1\n1000:0001\t3\n1000:0004\t1\n", data=declared_call,
|
|
453
|
+
noReturn=exit_routine, indirectCalls=[{"site": 1, "exhaustive": True, "targets": [4],
|
|
454
|
+
"evidence": "synthetic: bx holds 0004"}])
|
|
455
|
+
self.assertEqual((r["noReturn"][0]["returnSites"], r["noReturn"][0]["contradicted"]), ([3], True))
|
|
456
|
+
|
|
434
457
|
def test_a_contested_instruction_holds_no_target(self):
|
|
435
458
|
# Entries 0 and 1 overlap, so the call at 0003 is contested. The raw call at 000D targets 0004, a byte of
|
|
436
459
|
# that contested call, and names no holder, since only established instructions count.
|
|
@@ -604,15 +604,16 @@ class StopTests(unittest.TestCase):
|
|
|
604
604
|
self.assertEqual((r["noReturn"][0]["returnSites"], r["noReturn"][0]["contradicted"]), ([7], True))
|
|
605
605
|
self.assertTrue(r["noReturn"][0]["read"])
|
|
606
606
|
self.assertFalse(r["negativeUsable"])
|
|
607
|
-
# The check has the walk's instruction limit
|
|
608
|
-
#
|
|
609
|
-
|
|
610
|
-
|
|
607
|
+
# The check has the walk's instruction limit, spent on the routine's own paths. The call to the
|
|
608
|
+
# stop routine S ends the query's branch, so R's three instructions past it and its spin are read
|
|
609
|
+
# by the check alone, and a limit of three stops the check there.
|
|
610
|
+
data = bytes.fromhex("e80100" "c3" "e80500" "909090" "ebfe" "c3")
|
|
611
|
+
stop = [{"routine": 0x0C, "reason": CUT}]
|
|
611
612
|
r = run_report(data, config(data, targets=[3], controls=[0], noReturn=no_return, stops=stop), "reach")
|
|
612
613
|
self.assertEqual((r["noReturn"][0]["contradicted"], r["instructionLimitReached"], r["negativeUsable"]),
|
|
613
614
|
(False, False, True))
|
|
614
615
|
r = run_report(data, config(data, targets=[3], controls=[0], noReturn=no_return, stops=stop,
|
|
615
|
-
instructionLimit=
|
|
616
|
+
instructionLimit=3), "reach")
|
|
616
617
|
self.assertEqual(r["counts"]["instructions"], 2)
|
|
617
618
|
self.assertTrue(r["instructionLimitReached"])
|
|
618
619
|
self.assertFalse(r["negativeUsable"])
|
|
@@ -631,6 +632,80 @@ class StopTests(unittest.TestCase):
|
|
|
631
632
|
self.assertEqual([row["site"] for row in r["returnCheckGaps"]], [5])
|
|
632
633
|
self.assertFalse(r["negativeUsable"])
|
|
633
634
|
|
|
635
|
+
def test_the_no_return_check_reads_only_the_routines_own_paths(self):
|
|
636
|
+
# 0000 call R at 0004; 0003 ret. R calls the stop routine S at 000B and spins at 0007. S holds
|
|
637
|
+
# a computed jump nothing resolves, so a check that entered it would gain a returnCheckGaps row.
|
|
638
|
+
data = bytes.fromhex("e80100" "c3" "e80400" "ebfe" "9090" "90" "ffe0" "c3")
|
|
639
|
+
no_return = [{"routine": 4, "reason": "synthetic: declared to end the program"}]
|
|
640
|
+
stops = [{"routine": 0x0B, "reason": CUT}]
|
|
641
|
+
r = run_report(data, config(data, targets=[3], controls=[0], noReturn=no_return, stops=stops), "reach")
|
|
642
|
+
row = r["noReturn"][0]
|
|
643
|
+
self.assertEqual((row["returnSites"], row["contradicted"], row["read"]), ([], False, True))
|
|
644
|
+
self.assertEqual(r["returnCheckGaps"], [])
|
|
645
|
+
self.assertFalse(r["instructionLimitReached"])
|
|
646
|
+
self.assertTrue(r["negativeUsable"])
|
|
647
|
+
# A return on the routine's own path past the stepped-over call still contradicts the declaration.
|
|
648
|
+
returning = bytes.fromhex("e80100" "c3" "e80400" "c3" "909090" "90" "ffe0" "c3")
|
|
649
|
+
r = run_report(returning, config(returning, targets=[3], controls=[0], noReturn=no_return, stops=stops),
|
|
650
|
+
"reach")
|
|
651
|
+
self.assertEqual((r["noReturn"][0]["returnSites"], r["noReturn"][0]["contradicted"]), ([7], True))
|
|
652
|
+
self.assertEqual(r["returnCheckGaps"], [])
|
|
653
|
+
# A call to its own next instruction keeps its return site on the routine's path. The two arms
|
|
654
|
+
# of the branch at 0000 call the stop routine S at 000A and the declared routine R at 000B.
|
|
655
|
+
self_call = bytes.fromhex("7503" "e80500" "e80300" "c3" "90" "c3" "e80000" "c3")
|
|
656
|
+
r = run_report(self_call, config(self_call, targets=[8], controls=[2, 5],
|
|
657
|
+
noReturn=[{"routine": 0x0B, "reason": "synthetic: declared to end the program"}],
|
|
658
|
+
stops=[{"routine": 0x0A, "reason": CUT}]), "reach")
|
|
659
|
+
self.assertEqual((r["noReturn"][0]["returnSites"], r["noReturn"][0]["contradicted"]), ([0x0E], True))
|
|
660
|
+
|
|
661
|
+
def test_a_call_to_another_declared_routine_still_ends_the_checks_branch(self):
|
|
662
|
+
# The two arms of the branch at 0000 call the stop routine S at 000A and R1 at 000B. R1 calls
|
|
663
|
+
# R2 at 000F, both declared, with a return instruction after the call that is on no path of
|
|
664
|
+
# R1's own.
|
|
665
|
+
data = bytes.fromhex("7503" "e80500" "e80300" "c3" "90" "c3" "e80100" "c3" "ebfe")
|
|
666
|
+
no_return = [{"routine": 0x0B, "reason": "synthetic: declared to end the program"},
|
|
667
|
+
{"routine": 0x0F, "reason": "synthetic: declared to end the program"}]
|
|
668
|
+
r = run_report(data, config(data, targets=[8], controls=[2, 5], noReturn=no_return,
|
|
669
|
+
stops=[{"routine": 0x0A, "reason": CUT}]), "reach")
|
|
670
|
+
self.assertEqual([(row["returnSites"], row["contradicted"]) for row in r["noReturn"]],
|
|
671
|
+
[([], False), ([], False)])
|
|
672
|
+
self.assertEqual(r["returnCheckGaps"], [])
|
|
673
|
+
self.assertTrue(r["negativeUsable"])
|
|
674
|
+
|
|
675
|
+
def test_the_check_reads_past_a_stepped_over_declared_indirect_call(self):
|
|
676
|
+
# 0000 call R at 0004; 0003 ret. R calls the stop routine S at 000C, then makes a declared call
|
|
677
|
+
# at 0007 to T at 000D and returns at 0009, which only the check reads.
|
|
678
|
+
data = bytes.fromhex("e80100" "c3" "e80500" "ffd3" "c3" "9090" "c3" "c3")
|
|
679
|
+
no_return = [{"routine": 4, "reason": "synthetic: declared to end the program"}]
|
|
680
|
+
declared = [{"site": 7, "exhaustive": True, "targets": [0x0D], "evidence": "synthetic: bx holds 000D"}]
|
|
681
|
+
r = run_report(data, config(data, targets=[3], controls=[0], noReturn=no_return, indirectCalls=declared,
|
|
682
|
+
stops=[{"routine": 0x0C, "reason": CUT}]), "reach")
|
|
683
|
+
self.assertEqual((r["noReturn"][0]["returnSites"], r["noReturn"][0]["contradicted"]), ([9], True))
|
|
684
|
+
self.assertFalse(r["negativeUsable"])
|
|
685
|
+
|
|
686
|
+
def test_the_check_enters_a_declared_routine_the_query_reaches_only_past_a_stop(self):
|
|
687
|
+
# R1 at 0004 calls the stop routine S at 000D, then R2 at 000E, which returns. Only the check
|
|
688
|
+
# reaches R2, and its return contradicts R2's declaration.
|
|
689
|
+
data = bytes.fromhex("e80100" "c3" "e80600" "e80400" "ebfe" "90" "c3" "c3")
|
|
690
|
+
no_return = [{"routine": 4, "reason": "synthetic: declared to end the program"},
|
|
691
|
+
{"routine": 0x0E, "reason": "synthetic: declared to end the program"}]
|
|
692
|
+
r = run_report(data, config(data, targets=[3], controls=[0], noReturn=no_return,
|
|
693
|
+
stops=[{"routine": 0x0D, "reason": CUT}]), "reach")
|
|
694
|
+
self.assertEqual([(row["reached"], row["read"], row["returnSites"], row["contradicted"]) for row in r["noReturn"]],
|
|
695
|
+
[(True, True, [], False), (False, True, [0x0E], True)])
|
|
696
|
+
self.assertFalse(r["negativeUsable"])
|
|
697
|
+
|
|
698
|
+
def test_the_no_return_checks_callees_do_not_spend_the_instruction_limit(self):
|
|
699
|
+
# R's callee at 000B holds nine instructions; a check that entered it would trip a limit of
|
|
700
|
+
# six, where R's own path holds two.
|
|
701
|
+
data = bytes.fromhex("e80100" "c3" "e80400" "ebfe" "9090" + "90" * 8 + "c3")
|
|
702
|
+
no_return = [{"routine": 4, "reason": "synthetic: declared to end the program"}]
|
|
703
|
+
r = run_report(data, config(data, targets=[3], controls=[0], noReturn=no_return,
|
|
704
|
+
stops=[{"routine": 0x0B, "reason": CUT}], instructionLimit=6), "reach")
|
|
705
|
+
self.assertFalse(r["instructionLimitReached"])
|
|
706
|
+
self.assertFalse(r["noReturn"][0]["contradicted"])
|
|
707
|
+
self.assertTrue(r["negativeUsable"])
|
|
708
|
+
|
|
634
709
|
def test_rejected_stops(self):
|
|
635
710
|
site = {"site": 3, "reason": CUT}
|
|
636
711
|
for extra, message in [({"stops": {}}, "stops must be a list"),
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""Persistent scans use synthetic bytes only and never qualify raw candidates."""
|
|
2
|
+
import copy
|
|
3
|
+
from contextlib import closing
|
|
4
|
+
import os
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import sqlite3
|
|
7
|
+
import sys
|
|
8
|
+
import tempfile
|
|
9
|
+
import unittest
|
|
10
|
+
from unittest.mock import patch
|
|
11
|
+
|
|
12
|
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
|
13
|
+
from scientific_method_engine.x86.reports import run_report
|
|
14
|
+
from scientific_method_engine.x86.image import Image
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ScanIndexTests(unittest.TestCase):
|
|
18
|
+
def setUp(self):
|
|
19
|
+
self.folder = tempfile.TemporaryDirectory()
|
|
20
|
+
self.addCleanup(self.folder.cleanup)
|
|
21
|
+
# MOV AX,1234; MOV AX,[BX+44]; CALL 000C; CALL AX; RET.
|
|
22
|
+
self.data = bytes.fromhex("b834128b4744e80300ffd0c3c3")
|
|
23
|
+
self.config = {"sourceKind": "synthetic-raw", "regions": [
|
|
24
|
+
{"name": "code", "start": 0, "end": len(self.data), "ip": 0,
|
|
25
|
+
"segment": 4096, "entries": [0], "evidence": "synthetic"}],
|
|
26
|
+
"scanIndex": {"directory": self.folder.name},
|
|
27
|
+
"query": {"kind": "immediate", "value": 0x1234, "controls": [0]}}
|
|
28
|
+
|
|
29
|
+
def report(self, config=None, data=None):
|
|
30
|
+
return run_report(self.data if data is None else data, config or self.config, "scan-index")
|
|
31
|
+
|
|
32
|
+
def test_reuses_corpus_for_different_operands_without_decoding(self):
|
|
33
|
+
first = self.report()
|
|
34
|
+
self.assertFalse(first["cacheHit"])
|
|
35
|
+
self.assertEqual(first["rawCandidates"], [{"site": 0, "size": 3}])
|
|
36
|
+
config = copy.deepcopy(self.config)
|
|
37
|
+
config["query"] = {"kind": "displacement", "value": 0x44, "controls": [3]}
|
|
38
|
+
with patch.object(Image, "decode", side_effect=AssertionError("cache hit decoded again")):
|
|
39
|
+
second = self.report(config)
|
|
40
|
+
self.assertTrue(second["cacheHit"])
|
|
41
|
+
self.assertEqual(first["cacheKey"], second["cacheKey"])
|
|
42
|
+
self.assertEqual(second["rawCandidates"], [{"site": 3, "size": 3}])
|
|
43
|
+
|
|
44
|
+
def test_calls_controls_and_result_limit(self):
|
|
45
|
+
config = copy.deepcopy(self.config)
|
|
46
|
+
config["query"] = {"kind": "call", "value": 12, "controls": [6]}
|
|
47
|
+
self.assertIn({"site": 6, "size": 3}, self.report(config)["rawCandidates"])
|
|
48
|
+
config["query"]["controls"] = [0]
|
|
49
|
+
with self.assertRaisesRegex(ValueError, "controls missed"):
|
|
50
|
+
self.report(config)
|
|
51
|
+
repeated = bytes.fromhex("b83412b83412c3")
|
|
52
|
+
config["regions"][0]["end"] = len(repeated)
|
|
53
|
+
config["query"] = {"kind": "immediate", "value": 0x1234, "limit": 1}
|
|
54
|
+
report = self.report(config, repeated)
|
|
55
|
+
self.assertTrue(report["resultLimitReached"])
|
|
56
|
+
self.assertEqual(report["totalCandidates"], 2)
|
|
57
|
+
self.assertEqual(len(report["rawCandidates"]), 1)
|
|
58
|
+
|
|
59
|
+
def test_reports_where_the_scan_could_not_see(self):
|
|
60
|
+
report = self.report()
|
|
61
|
+
self.assertEqual(report["unresolvedCalls"], [{"site": 9, "size": 2}])
|
|
62
|
+
self.assertFalse(report["unresolvedCallsLimitReached"])
|
|
63
|
+
self.assertEqual(report["undecodedRanges"], [])
|
|
64
|
+
# RET, then MOV AX,imm16 and XOR AL,imm8 cut off by each region's end.
|
|
65
|
+
data = bytes.fromhex("c3b834c3b834")
|
|
66
|
+
config = copy.deepcopy(self.config)
|
|
67
|
+
config["regions"] = [
|
|
68
|
+
{"name": "a", "start": 0, "end": 3, "ip": 0, "segment": 4096, "entries": [0], "evidence": "synthetic"},
|
|
69
|
+
{"name": "b", "start": 3, "end": 6, "ip": 0, "segment": 8192, "entries": [3], "evidence": "synthetic"}]
|
|
70
|
+
config["query"] = {"kind": "immediate", "value": 0x1234}
|
|
71
|
+
first = self.report(config, data)
|
|
72
|
+
self.assertEqual(first["undecodedRanges"], [{"start": 1, "end": 3}, {"start": 4, "end": 6}])
|
|
73
|
+
self.assertEqual(first["scan"]["undecodedOffsets"], 4)
|
|
74
|
+
config["query"]["limit"] = 1
|
|
75
|
+
with patch.object(Image, "decode", side_effect=AssertionError("cache hit decoded again")):
|
|
76
|
+
capped = self.report(config, data)
|
|
77
|
+
self.assertTrue(capped["cacheHit"])
|
|
78
|
+
self.assertEqual(capped["undecodedRanges"], [{"start": 1, "end": 3}])
|
|
79
|
+
self.assertTrue(capped["undecodedRangesLimitReached"])
|
|
80
|
+
|
|
81
|
+
def test_changed_source_mapping_entries_and_decoder_identity_invalidate(self):
|
|
82
|
+
first = self.report()["cacheKey"]
|
|
83
|
+
self.assertNotEqual(first, self.report(data=self.data[:-1] + b"\x90")["cacheKey"])
|
|
84
|
+
for field, value in (("ip", 32), ("entries", [3]), ("evidence", "changed mapping")):
|
|
85
|
+
config = copy.deepcopy(self.config)
|
|
86
|
+
config["regions"][0][field] = value
|
|
87
|
+
self.assertNotEqual(first, self.report(config)["cacheKey"])
|
|
88
|
+
with patch("scientific_method_engine.x86.scan_index.CAPSTONE_VERSION", "different"):
|
|
89
|
+
self.assertNotEqual(first, self.report()["cacheKey"])
|
|
90
|
+
|
|
91
|
+
def test_incomplete_scan_writes_nothing_and_invalid_options_fail(self):
|
|
92
|
+
config = copy.deepcopy(self.config)
|
|
93
|
+
config["scanLimit"] = 2
|
|
94
|
+
with self.assertRaisesRegex(ValueError, "scanLimit reached"):
|
|
95
|
+
self.report(config)
|
|
96
|
+
self.assertEqual(list(Path(self.folder.name).iterdir()), [])
|
|
97
|
+
config = copy.deepcopy(self.config)
|
|
98
|
+
config["scanIndex"]["directory"] = "relative"
|
|
99
|
+
with self.assertRaisesRegex(ValueError, "absolute"):
|
|
100
|
+
self.report(config)
|
|
101
|
+
for query in ({"kind": "unknown", "value": 0}, {"kind": "call", "value": True},
|
|
102
|
+
{"kind": "call", "value": 0, "limit": 0},
|
|
103
|
+
{"kind": "call", "value": 0, "limit": 10001},
|
|
104
|
+
{"kind": "call", "value": 0, "controls": [0] * 129},
|
|
105
|
+
{"kind": "call", "value": 0, "controls": [999]}):
|
|
106
|
+
config = copy.deepcopy(self.config)
|
|
107
|
+
config["query"] = query
|
|
108
|
+
with self.assertRaises(ValueError):
|
|
109
|
+
self.report(config)
|
|
110
|
+
|
|
111
|
+
def test_corrupt_or_mismatched_cache_is_refused(self):
|
|
112
|
+
report = self.report()
|
|
113
|
+
path = Path(self.folder.name) / (report["cacheKey"] + ".sqlite")
|
|
114
|
+
with closing(sqlite3.connect(path)) as db, db:
|
|
115
|
+
db.execute("UPDATE metadata SET value='{}'")
|
|
116
|
+
with self.assertRaisesRegex(ValueError, "identity differs"):
|
|
117
|
+
self.report()
|
|
118
|
+
path.write_bytes(b"not a database")
|
|
119
|
+
with self.assertRaisesRegex(ValueError, "cache failed"):
|
|
120
|
+
self.report()
|
|
121
|
+
|
|
122
|
+
def test_implicit_operands_branch_targets_and_prefixed_calls_are_not_candidates(self):
|
|
123
|
+
# MOV AX,[BX]; SHL AX,1; JE 0006; CALL with an operand-size override to 000C; RET.
|
|
124
|
+
data = bytes.fromhex("8b07d1e0740066e800000000c3")
|
|
125
|
+
config = copy.deepcopy(self.config)
|
|
126
|
+
config["regions"][0]["end"] = len(data)
|
|
127
|
+
for kind, value in (("displacement", 0), ("immediate", 1), ("immediate", 6), ("call", 12)):
|
|
128
|
+
config["query"] = {"kind": kind, "value": value}
|
|
129
|
+
self.assertEqual(self.report(config, data)["totalCandidates"], 0, (kind, value))
|
|
130
|
+
# The unprefixed CALL decoded inside the prefixed one still resolves.
|
|
131
|
+
config["query"] = {"kind": "call", "value": 10, "controls": [7]}
|
|
132
|
+
self.assertEqual(self.report(config, data)["rawCandidates"], [{"site": 7, "size": 3}])
|
|
133
|
+
|
|
134
|
+
def test_flat32_has_separate_mapping_and_candidates(self):
|
|
135
|
+
data = bytes.fromhex("b878563412c3")
|
|
136
|
+
config = copy.deepcopy(self.config)
|
|
137
|
+
config.update(bits=32, addressModel="flat32")
|
|
138
|
+
config["regions"][0].update(end=len(data), segment=0, ip=0x1000)
|
|
139
|
+
config["query"] = {"kind": "immediate", "value": 0x12345678, "controls": [0]}
|
|
140
|
+
self.assertEqual(self.report(config, data)["rawCandidates"], [{"site": 0, "size": 5}])
|
|
141
|
+
|
|
142
|
+
def test_concurrent_publication_of_the_same_corpus_is_kept(self):
|
|
143
|
+
# Windows refuses the replace while another run has the published corpus open.
|
|
144
|
+
replace = os.replace
|
|
145
|
+
|
|
146
|
+
def published_by_another_run(source, destination):
|
|
147
|
+
replace(source, destination)
|
|
148
|
+
raise PermissionError("in use")
|
|
149
|
+
with patch("scientific_method_engine.x86.scan_index.os.replace", side_effect=published_by_another_run):
|
|
150
|
+
report = self.report()
|
|
151
|
+
self.assertEqual(report["rawCandidates"], [{"site": 0, "size": 3}])
|
|
152
|
+
self.assertEqual(list(Path(self.folder.name).glob('*.partial')), [])
|
|
153
|
+
|
|
154
|
+
def test_disk_budget_failure_never_publishes_a_partial_index(self):
|
|
155
|
+
with patch("scientific_method_engine.x86.scan_index.MAX_INDEX_BYTES", 4096):
|
|
156
|
+
with self.assertRaisesRegex(ValueError, "build failed and published nothing"):
|
|
157
|
+
self.report()
|
|
158
|
+
self.assertEqual(list(Path(self.folder.name).glob('*.sqlite')), [])
|
|
159
|
+
self.assertEqual(list(Path(self.folder.name).glob('*.partial')), [])
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
if __name__ == "__main__":
|
|
163
|
+
unittest.main()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_config_encoding.py
RENAMED
|
File without changes
|
|
File without changes
|
{scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_derived_counts.py
RENAMED
|
File without changes
|
|
File without changes
|
{scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_effect_order.py
RENAMED
|
File without changes
|
{scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_enter_frames.py
RENAMED
|
File without changes
|
{scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_entry_frame.py
RENAMED
|
File without changes
|
|
File without changes
|
{scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_interrupt_models.py
RENAMED
|
File without changes
|
{scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_join_terms.py
RENAMED
|
File without changes
|
|
File without changes
|
{scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_memory_scopes.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_return_checks.py
RENAMED
|
File without changes
|
{scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_successor_rule.py
RENAMED
|
File without changes
|
{scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_table_bounds.py
RENAMED
|
File without changes
|
|
File without changes
|
{scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_term_limit.py
RENAMED
|
File without changes
|
{scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_upstream_gaps.py
RENAMED
|
File without changes
|
{scientific_method_engine-19.8.0 → scientific_method_engine-19.9.0}/tests/test_volatile_memory.py
RENAMED
|
File without changes
|
|
File without changes
|