scientific-method-engine 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- scientific_method_engine-0.1.0/.gitignore +20 -0
- scientific_method_engine-0.1.0/LICENSE +21 -0
- scientific_method_engine-0.1.0/PKG-INFO +110 -0
- scientific_method_engine-0.1.0/README.md +98 -0
- scientific_method_engine-0.1.0/pyproject.toml +26 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/__init__.py +5 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/__main__.py +3 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/cli.py +75 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +22 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/CreateFunctions.java +27 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +59 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +135 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +50 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +63 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +93 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/RepairReturningCallers.java +76 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportCallArguments.java +65 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportCallPaths.java +106 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +95 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportCallsToRange.java +67 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +55 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportDataBytes.java +36 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +71 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +61 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +102 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +63 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +56 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +64 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportInstructionContext.java +64 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +36 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +105 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +52 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +74 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportReferences.java +42 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportScalarConstants.java +55 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportStringReferences.java +102 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +72 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/x86/__init__.py +0 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/x86/dispatch.py +51 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/x86/image.py +172 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/x86/machine.py +659 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/x86/pe.py +96 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/x86/reports.py +1259 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/x86/trace.py +547 -0
- scientific_method_engine-0.1.0/src/scientific_method_engine/x86/values.py +123 -0
- scientific_method_engine-0.1.0/tests/test_dispatch.py +106 -0
- scientific_method_engine-0.1.0/tests/test_pe.py +437 -0
- scientific_method_engine-0.1.0/tests/test_x86.py +1389 -0
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
.config/
|
|
2
|
+
.idea/
|
|
3
|
+
.vs/
|
|
4
|
+
**/bin/
|
|
5
|
+
**/obj/
|
|
6
|
+
artifacts/
|
|
7
|
+
template-next/
|
|
8
|
+
TestResults/
|
|
9
|
+
*.user
|
|
10
|
+
*.suo
|
|
11
|
+
|
|
12
|
+
__pycache__/
|
|
13
|
+
*.pyc
|
|
14
|
+
.venv/
|
|
15
|
+
analysis/original/
|
|
16
|
+
node_modules/
|
|
17
|
+
dist/
|
|
18
|
+
packages/*/LICENSE
|
|
19
|
+
# The reader's command-line entry point is source, unlike .NET build output.
|
|
20
|
+
!packages/executable-reader/bin/
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Igor Savin
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: scientific-method-engine
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
|
|
5
|
+
Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
|
|
6
|
+
Author: kibertoad
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
|
+
Requires-Dist: capstone==5.0.7
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
|
|
13
|
+
# scientific-method-engine
|
|
14
|
+
|
|
15
|
+
Bounded instruction-derived x86 evidence reports for segmented 16-bit MZ/FBOV code and
|
|
16
|
+
PE32/i386 code. The engine decodes instructions with Capstone, follows bounded paths and emits
|
|
17
|
+
`bounded-x86-v1` JSON. It never runs the original program.
|
|
18
|
+
|
|
19
|
+
```sh
|
|
20
|
+
uv add --group research scientific-method-engine # or: pip install scientific-method-engine
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
For original MZ/FBOV executables, run reports through
|
|
24
|
+
[`@scientific-method/executable-reader`](https://www.npmjs.com/package/@scientific-method/executable-reader). The
|
|
25
|
+
reader derives relocation, fixup and trampoline data from the hash-checked source and pipes a
|
|
26
|
+
prepared config to this engine. Running the engine directly trusts whatever relocation data the
|
|
27
|
+
config supplies, so use it directly only for synthetic inputs, PE32 sources and checked mappings:
|
|
28
|
+
|
|
29
|
+
```sh
|
|
30
|
+
scientific-method-engine trace analysis/query.json
|
|
31
|
+
python -m scientific_method_engine trace analysis/query.json
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
The package also carries the shared Ghidra headless scripts. `scientific-method-engine
|
|
35
|
+
ghidra-scripts` prints their directory, for Ghidra's `-scriptPath`:
|
|
36
|
+
|
|
37
|
+
```powershell
|
|
38
|
+
& "$env:GHIDRA_HOME/support/analyzeHeadless.bat" $project $name -process GAME.EXE -noanalysis `
|
|
39
|
+
-scriptPath (scientific-method-engine ghidra-scripts) -postScript ReportReferences.java 0x1234
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
Addresses are Ghidra addresses (`0x00401000`, or `1028:d820` for segmented programs). Report scripts
|
|
43
|
+
print to the analyzer log and cap their output. The scripts compile against Ghidra 12.1.
|
|
44
|
+
|
|
45
|
+
Reading code and data:
|
|
46
|
+
|
|
47
|
+
| Script | Arguments | Prints |
|
|
48
|
+
|---|---|---|
|
|
49
|
+
| `ReportInstructionContext` | one or more instruction addresses | a bounded instruction window around each |
|
|
50
|
+
| `ReportInstructionWindow` | address, instruction count | instructions from the address onward |
|
|
51
|
+
| `ReportDataBytes` | address, byte count (1..256) | the bytes at the address |
|
|
52
|
+
| `ReportFunctionSummary` | one or more addresses | focused decompiler output of each containing function |
|
|
53
|
+
| `ReportDecompileWindow` | address, first line (1-based), line count | a window of one function's decompilation |
|
|
54
|
+
| `ReportDecompileMatches` | address, one or more literal text patterns | decompilation lines around each match |
|
|
55
|
+
| `ReportMemoryBlocks` | nothing, `page <start> <count>`, or `name <exact-name>` | memory block indexes, names, ranges and sizes, never bytes |
|
|
56
|
+
| `ReportFilePatternInMemory` | file offset (hex), optional pattern length (default 8) | where the bytes at that file offset occur in loaded memory |
|
|
57
|
+
| `ReportMemoryBlockForFileOffset` | one or more file offsets (hex) | the memory block and instructions where each offset's bytes are loaded |
|
|
58
|
+
|
|
59
|
+
Finding references and calls:
|
|
60
|
+
|
|
61
|
+
| Script | Arguments | Prints |
|
|
62
|
+
|---|---|---|
|
|
63
|
+
| `ReportReferences` | one or more addresses | references to each, with the referring instruction and function |
|
|
64
|
+
| `ReportStringReferences` | one or more literal string fragments | strings containing a fragment and their references |
|
|
65
|
+
| `ReportSymbolReferences` | one or more symbol-name fragments | matching symbols and their references |
|
|
66
|
+
| `ReportScalarConstants` | one or more scalar values | instructions using any of them, unsigned or signed |
|
|
67
|
+
| `ReportFunctionScalarConstants` | function address, one or more scalar values | instructions inside one function using any of them, compared unsigned |
|
|
68
|
+
| `ReportCallArguments` | callee address | the three nearest pushed arguments at every direct call |
|
|
69
|
+
| `ReportCallSitesWithScalars` | callee address, one or more scalar values | calls whose argument setup contains a requested value |
|
|
70
|
+
| `ReportConstantFirstArgumentCalls` | callee address, constant | cdecl calls whose first argument is the constant |
|
|
71
|
+
| `ReportFirstArgumentCallSummary` | callee address | the literal first argument of every call, and calls without one |
|
|
72
|
+
| `ReportCallsToRange` | start address, end address (inclusive) | calls and jumps whose target lies in the range |
|
|
73
|
+
| `ReportCallPaths` | start function, target function, maximum depth | direct-call paths between the two |
|
|
74
|
+
| `ReportRandomnessCandidates` | none | references to C runtime and Windows random and timing functions |
|
|
75
|
+
|
|
76
|
+
Exporting for comparison (each writes one file and refuses to overwrite where noted):
|
|
77
|
+
|
|
78
|
+
| Script | Arguments | Writes |
|
|
79
|
+
|---|---|---|
|
|
80
|
+
| `ExportBoundedFlow` | entry, instruction limit (1..10000), output path under `analysis/original/` | instruction metadata of one bounded flow as JSON |
|
|
81
|
+
| `ExportFunctionInventory` | output TSV path (must not exist) | every function's start and body size |
|
|
82
|
+
| `ExportFunctionFingerprints` | output TSV path | per-function and per-instruction fingerprints with addresses normalized, for matching functions across versions |
|
|
83
|
+
|
|
84
|
+
Repairing the analysis (these change the Ghidra program, so run them before reports and keep the
|
|
85
|
+
argument lists with the evidence that justifies them):
|
|
86
|
+
|
|
87
|
+
| Script | Arguments | Changes |
|
|
88
|
+
|---|---|---|
|
|
89
|
+
| `CreateFunctions` | one or more entry addresses | creates functions at indirect-call targets Ghidra missed |
|
|
90
|
+
| `RecoverCitedFunctions` | file of 8-digit hex addresses, optional CSV column | disassembles and creates a function at each address inside executable memory |
|
|
91
|
+
| `ClearNoReturnFunctions` | one or more addresses | clears a wrong no-return flag on each containing function |
|
|
92
|
+
| `RepairReturningCallers` | callee entry, caller entry, verified call addresses | checks each call targets the callee from inside the caller, clears the callee's no-return flag and the calls' flow overrides, disassembles each continuation and recomputes the caller's body |
|
|
93
|
+
| `MergeFallThroughFragment` | parent entry, fragment entry | merges an orphan fragment reached by the parent's fall-through into the parent |
|
|
94
|
+
|
|
95
|
+
Restoration tools that build report configs can import the parsers directly, for example to
|
|
96
|
+
derive a PE32 source's executable sections:
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
from scientific_method_engine.x86.pe import pe32
|
|
100
|
+
|
|
101
|
+
sections = [s for s in pe32(data)["sections"] if s["executable"]]
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
`scientific_method_engine.x86.pe.pe32` and `scientific_method_engine.x86.image.read_source` are
|
|
105
|
+
supported imports and change only in a major release. Other modules are internal.
|
|
106
|
+
|
|
107
|
+
Commands, inputs, limits and acceptance rules are in
|
|
108
|
+
[the bounded evidence reporter guide](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md).
|
|
109
|
+
The reader and engine check that they speak the same prepared-config protocol and refuse to run
|
|
110
|
+
otherwise.
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# scientific-method-engine
|
|
2
|
+
|
|
3
|
+
Bounded instruction-derived x86 evidence reports for segmented 16-bit MZ/FBOV code and
|
|
4
|
+
PE32/i386 code. The engine decodes instructions with Capstone, follows bounded paths and emits
|
|
5
|
+
`bounded-x86-v1` JSON. It never runs the original program.
|
|
6
|
+
|
|
7
|
+
```sh
|
|
8
|
+
uv add --group research scientific-method-engine # or: pip install scientific-method-engine
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
For original MZ/FBOV executables, run reports through
|
|
12
|
+
[`@scientific-method/executable-reader`](https://www.npmjs.com/package/@scientific-method/executable-reader). The
|
|
13
|
+
reader derives relocation, fixup and trampoline data from the hash-checked source and pipes a
|
|
14
|
+
prepared config to this engine. Running the engine directly trusts whatever relocation data the
|
|
15
|
+
config supplies, so use it directly only for synthetic inputs, PE32 sources and checked mappings:
|
|
16
|
+
|
|
17
|
+
```sh
|
|
18
|
+
scientific-method-engine trace analysis/query.json
|
|
19
|
+
python -m scientific_method_engine trace analysis/query.json
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
The package also carries the shared Ghidra headless scripts. `scientific-method-engine
|
|
23
|
+
ghidra-scripts` prints their directory, for Ghidra's `-scriptPath`:
|
|
24
|
+
|
|
25
|
+
```powershell
|
|
26
|
+
& "$env:GHIDRA_HOME/support/analyzeHeadless.bat" $project $name -process GAME.EXE -noanalysis `
|
|
27
|
+
-scriptPath (scientific-method-engine ghidra-scripts) -postScript ReportReferences.java 0x1234
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Addresses are Ghidra addresses (`0x00401000`, or `1028:d820` for segmented programs). Report scripts
|
|
31
|
+
print to the analyzer log and cap their output. The scripts compile against Ghidra 12.1.
|
|
32
|
+
|
|
33
|
+
Reading code and data:
|
|
34
|
+
|
|
35
|
+
| Script | Arguments | Prints |
|
|
36
|
+
|---|---|---|
|
|
37
|
+
| `ReportInstructionContext` | one or more instruction addresses | a bounded instruction window around each |
|
|
38
|
+
| `ReportInstructionWindow` | address, instruction count | instructions from the address onward |
|
|
39
|
+
| `ReportDataBytes` | address, byte count (1..256) | the bytes at the address |
|
|
40
|
+
| `ReportFunctionSummary` | one or more addresses | focused decompiler output of each containing function |
|
|
41
|
+
| `ReportDecompileWindow` | address, first line (1-based), line count | a window of one function's decompilation |
|
|
42
|
+
| `ReportDecompileMatches` | address, one or more literal text patterns | decompilation lines around each match |
|
|
43
|
+
| `ReportMemoryBlocks` | nothing, `page <start> <count>`, or `name <exact-name>` | memory block indexes, names, ranges and sizes, never bytes |
|
|
44
|
+
| `ReportFilePatternInMemory` | file offset (hex), optional pattern length (default 8) | where the bytes at that file offset occur in loaded memory |
|
|
45
|
+
| `ReportMemoryBlockForFileOffset` | one or more file offsets (hex) | the memory block and instructions where each offset's bytes are loaded |
|
|
46
|
+
|
|
47
|
+
Finding references and calls:
|
|
48
|
+
|
|
49
|
+
| Script | Arguments | Prints |
|
|
50
|
+
|---|---|---|
|
|
51
|
+
| `ReportReferences` | one or more addresses | references to each, with the referring instruction and function |
|
|
52
|
+
| `ReportStringReferences` | one or more literal string fragments | strings containing a fragment and their references |
|
|
53
|
+
| `ReportSymbolReferences` | one or more symbol-name fragments | matching symbols and their references |
|
|
54
|
+
| `ReportScalarConstants` | one or more scalar values | instructions using any of them, unsigned or signed |
|
|
55
|
+
| `ReportFunctionScalarConstants` | function address, one or more scalar values | instructions inside one function using any of them, compared unsigned |
|
|
56
|
+
| `ReportCallArguments` | callee address | the three nearest pushed arguments at every direct call |
|
|
57
|
+
| `ReportCallSitesWithScalars` | callee address, one or more scalar values | calls whose argument setup contains a requested value |
|
|
58
|
+
| `ReportConstantFirstArgumentCalls` | callee address, constant | cdecl calls whose first argument is the constant |
|
|
59
|
+
| `ReportFirstArgumentCallSummary` | callee address | the literal first argument of every call, and calls without one |
|
|
60
|
+
| `ReportCallsToRange` | start address, end address (inclusive) | calls and jumps whose target lies in the range |
|
|
61
|
+
| `ReportCallPaths` | start function, target function, maximum depth | direct-call paths between the two |
|
|
62
|
+
| `ReportRandomnessCandidates` | none | references to C runtime and Windows random and timing functions |
|
|
63
|
+
|
|
64
|
+
Exporting for comparison (each writes one file and refuses to overwrite where noted):
|
|
65
|
+
|
|
66
|
+
| Script | Arguments | Writes |
|
|
67
|
+
|---|---|---|
|
|
68
|
+
| `ExportBoundedFlow` | entry, instruction limit (1..10000), output path under `analysis/original/` | instruction metadata of one bounded flow as JSON |
|
|
69
|
+
| `ExportFunctionInventory` | output TSV path (must not exist) | every function's start and body size |
|
|
70
|
+
| `ExportFunctionFingerprints` | output TSV path | per-function and per-instruction fingerprints with addresses normalized, for matching functions across versions |
|
|
71
|
+
|
|
72
|
+
Repairing the analysis (these change the Ghidra program, so run them before reports and keep the
|
|
73
|
+
argument lists with the evidence that justifies them):
|
|
74
|
+
|
|
75
|
+
| Script | Arguments | Changes |
|
|
76
|
+
|---|---|---|
|
|
77
|
+
| `CreateFunctions` | one or more entry addresses | creates functions at indirect-call targets Ghidra missed |
|
|
78
|
+
| `RecoverCitedFunctions` | file of 8-digit hex addresses, optional CSV column | disassembles and creates a function at each address inside executable memory |
|
|
79
|
+
| `ClearNoReturnFunctions` | one or more addresses | clears a wrong no-return flag on each containing function |
|
|
80
|
+
| `RepairReturningCallers` | callee entry, caller entry, verified call addresses | checks each call targets the callee from inside the caller, clears the callee's no-return flag and the calls' flow overrides, disassembles each continuation and recomputes the caller's body |
|
|
81
|
+
| `MergeFallThroughFragment` | parent entry, fragment entry | merges an orphan fragment reached by the parent's fall-through into the parent |
|
|
82
|
+
|
|
83
|
+
Restoration tools that build report configs can import the parsers directly, for example to
|
|
84
|
+
derive a PE32 source's executable sections:
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
from scientific_method_engine.x86.pe import pe32
|
|
88
|
+
|
|
89
|
+
sections = [s for s in pe32(data)["sections"] if s["executable"]]
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
`scientific_method_engine.x86.pe.pe32` and `scientific_method_engine.x86.image.read_source` are
|
|
93
|
+
supported imports and change only in a major release. Other modules are internal.
|
|
94
|
+
|
|
95
|
+
Commands, inputs, limits and acceptance rules are in
|
|
96
|
+
[the bounded evidence reporter guide](https://github.com/kibertoad/refurbished-dinosaurs-toolkit/blob/main/docs/bounded-evidence-reporters.md).
|
|
97
|
+
The reader and engine check that they speak the same prepared-config protocol and refuse to run
|
|
98
|
+
otherwise.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "scientific-method-engine"
|
|
7
|
+
# The release workflow writes the published version from the package's release tag.
|
|
8
|
+
version = "0.1.0"
|
|
9
|
+
description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
|
|
10
|
+
readme = "README.md"
|
|
11
|
+
requires-python = ">=3.10"
|
|
12
|
+
license = "MIT"
|
|
13
|
+
authors = [{ name = "kibertoad" }]
|
|
14
|
+
dependencies = ["capstone==5.0.7"]
|
|
15
|
+
|
|
16
|
+
[project.scripts]
|
|
17
|
+
scientific-method-engine = "scientific_method_engine.cli:run"
|
|
18
|
+
|
|
19
|
+
[project.urls]
|
|
20
|
+
Source = "https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine"
|
|
21
|
+
|
|
22
|
+
[tool.hatch.build.targets.wheel]
|
|
23
|
+
packages = ["src/scientific_method_engine"]
|
|
24
|
+
|
|
25
|
+
[tool.hatch.build.targets.sdist]
|
|
26
|
+
include = ["src", "tests", "README.md", "pyproject.toml"]
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
"""Bounded instruction-derived x86 evidence reports."""
|
|
2
|
+
|
|
3
|
+
# The reader passes this number with every prepared config it pipes to the engine. A reader and
|
|
4
|
+
# engine that disagree on it refuse to run instead of exchanging relocation data in another shape.
|
|
5
|
+
PREPARED_PROTOCOL = 1
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
"""Read-only bounded x86 evidence reports. Keep configs and reports in GAME_DIR, uncommitted."""
|
|
2
|
+
import json
|
|
3
|
+
import sys
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from . import PREPARED_PROTOCOL
|
|
7
|
+
|
|
8
|
+
CONFIG_LIMIT = 1024 * 1024
|
|
9
|
+
PREPARED_CONFIG_LIMIT = 16 * 1024 * 1024
|
|
10
|
+
USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bounds|owner|callees|trace|uses|arguments|"
|
|
11
|
+
"effects|returns|memory|incoming|guards|allocation|dispatch> <config.json|->\n"
|
|
12
|
+
" scientific-method-engine ghidra-scripts")
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def ghidra_scripts():
|
|
16
|
+
"""The directory to pass to Ghidra's analyzeHeadless -scriptPath."""
|
|
17
|
+
return Path(__file__).resolve().parent / "ghidra"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def main(argv):
|
|
21
|
+
"""Run one command (``argv`` excludes the program name) and print its JSON report."""
|
|
22
|
+
from .x86.image import read_source
|
|
23
|
+
from .x86.reports import run_report
|
|
24
|
+
if argv == ["ghidra-scripts"]:
|
|
25
|
+
print(ghidra_scripts())
|
|
26
|
+
return
|
|
27
|
+
if len(argv) != 2:
|
|
28
|
+
raise ValueError(USAGE)
|
|
29
|
+
command, config_path = argv
|
|
30
|
+
reject_derived = False
|
|
31
|
+
if config_path == "-":
|
|
32
|
+
# The reader caps its input at 1 MiB, then adds every source relocation.
|
|
33
|
+
limit, label = PREPARED_CONFIG_LIMIT, "Prepared config exceeds 16 MiB"
|
|
34
|
+
text = sys.stdin.read(limit + 1)
|
|
35
|
+
base = Path.cwd()
|
|
36
|
+
else:
|
|
37
|
+
limit, label = CONFIG_LIMIT, "Config exceeds 1 MiB"
|
|
38
|
+
path = Path(config_path).resolve()
|
|
39
|
+
if path.stat().st_size > limit:
|
|
40
|
+
raise ValueError(label)
|
|
41
|
+
text, base = path.read_text(encoding="utf-8"), path.parent
|
|
42
|
+
reject_derived = True
|
|
43
|
+
if len(text.encode("utf-8")) > limit:
|
|
44
|
+
raise ValueError(label)
|
|
45
|
+
config = json.loads(text)
|
|
46
|
+
if not isinstance(config, dict):
|
|
47
|
+
raise ValueError("Config must be an object")
|
|
48
|
+
if reject_derived and "overlayExports" in config:
|
|
49
|
+
# Only the reader's MZ/FBOV loader (stdin mode) derives overlay exports from source tables.
|
|
50
|
+
raise ValueError("overlayExports is source-derived and cannot be supplied")
|
|
51
|
+
if config_path == "-":
|
|
52
|
+
protocol = config.pop("preparedProtocol", None)
|
|
53
|
+
if protocol != PREPARED_PROTOCOL:
|
|
54
|
+
raise ValueError(f"Reader sent prepared config protocol {protocol}; this engine reads protocol "
|
|
55
|
+
f"{PREPARED_PROTOCOL}. Install matching @scientific-method/executable-reader and "
|
|
56
|
+
"scientific-method-engine releases.")
|
|
57
|
+
elif "preparedProtocol" in config:
|
|
58
|
+
raise ValueError("preparedProtocol is set by the reader and cannot be supplied")
|
|
59
|
+
data, identity = read_source(config, base)
|
|
60
|
+
result = run_report(data, config, command)
|
|
61
|
+
print(json.dumps({"schema": "bounded-x86-v1", "decoder": "capstone 5.0.7", "sourceIdentity": identity,
|
|
62
|
+
"status": "Conditional static report; never promotes an evidence entry", **result}, indent=2))
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def run():
|
|
66
|
+
"""Entry point of the ``scientific-method-engine`` command; exits with 1 on any error."""
|
|
67
|
+
try:
|
|
68
|
+
main(sys.argv[1:])
|
|
69
|
+
except (ValueError, TypeError, KeyError, OSError, ImportError) as error:
|
|
70
|
+
print("Evidence report: " + str(error), file=sys.stderr)
|
|
71
|
+
sys.exit(1)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
if __name__ == "__main__":
|
|
75
|
+
run()
|
scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
// Clears incorrect no-return markings at explicitly supplied function addresses.
|
|
2
|
+
// @category Restoration
|
|
3
|
+
|
|
4
|
+
import ghidra.app.script.GhidraScript;
|
|
5
|
+
import ghidra.program.model.address.Address;
|
|
6
|
+
import ghidra.program.model.listing.Function;
|
|
7
|
+
|
|
8
|
+
public class ClearNoReturnFunctions extends GhidraScript {
|
|
9
|
+
@Override
|
|
10
|
+
protected void run() throws Exception {
|
|
11
|
+
for (String argument : getScriptArgs()) {
|
|
12
|
+
Address address = toAddr(argument);
|
|
13
|
+
Function function = getFunctionContaining(address);
|
|
14
|
+
if (function == null) {
|
|
15
|
+
printerr("No function contains " + address);
|
|
16
|
+
continue;
|
|
17
|
+
}
|
|
18
|
+
function.setNoReturn(false);
|
|
19
|
+
println("Cleared no-return on " + function.getName() + " at " + function.getEntryPoint());
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
// Creates functions at explicitly supplied indirect-call targets.
|
|
2
|
+
// @category Restoration
|
|
3
|
+
|
|
4
|
+
import ghidra.app.script.GhidraScript;
|
|
5
|
+
import ghidra.program.model.address.Address;
|
|
6
|
+
import ghidra.program.model.listing.Function;
|
|
7
|
+
|
|
8
|
+
public class CreateFunctions extends GhidraScript {
|
|
9
|
+
@Override
|
|
10
|
+
protected void run() throws Exception {
|
|
11
|
+
String[] arguments = getScriptArgs();
|
|
12
|
+
if (arguments.length == 0) {
|
|
13
|
+
printerr("Pass one or more function entry addresses.");
|
|
14
|
+
return;
|
|
15
|
+
}
|
|
16
|
+
for (String argument : arguments) {
|
|
17
|
+
Address address = toAddr(argument);
|
|
18
|
+
Function function = getFunctionAt(address);
|
|
19
|
+
if (function == null) function = createFunction(address, null);
|
|
20
|
+
if (function == null) {
|
|
21
|
+
printerr("Could not create function at " + address);
|
|
22
|
+
continue;
|
|
23
|
+
}
|
|
24
|
+
println("FUNCTION " + function.getName() + " " + function.getEntryPoint());
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// Exports instruction metadata for one entry. Output belongs in ignored analysis/original/.
|
|
2
|
+
// @category Restoration
|
|
3
|
+
import ghidra.app.script.GhidraScript;
|
|
4
|
+
import ghidra.program.model.address.Address;
|
|
5
|
+
import ghidra.program.model.listing.Instruction;
|
|
6
|
+
import ghidra.program.model.listing.Function;
|
|
7
|
+
import java.nio.file.*;
|
|
8
|
+
import java.nio.charset.StandardCharsets;
|
|
9
|
+
import java.util.*;
|
|
10
|
+
|
|
11
|
+
public class ExportBoundedFlow extends GhidraScript {
|
|
12
|
+
@Override protected void run() throws Exception {
|
|
13
|
+
String[] args = getScriptArgs();
|
|
14
|
+
if (args.length != 3) throw new IllegalArgumentException("Supply entry, instruction limit (1..10000), ignored analysis/original/output.json");
|
|
15
|
+
Address entry = toAddr(args[0]);
|
|
16
|
+
int limit = Integer.parseInt(args[1]);
|
|
17
|
+
if (limit < 1 || limit > 10000) throw new IllegalArgumentException("Limit must be 1..10000");
|
|
18
|
+
Path output = Path.of(args[2]).toAbsolutePath().normalize();
|
|
19
|
+
if (!output.toString().replace('\\', '/').contains("/analysis/original/")) throw new IllegalArgumentException("Flow reports stay under ignored analysis/original/");
|
|
20
|
+
Set<Address> visited = new HashSet<>();
|
|
21
|
+
Deque<Address> pending = new ArrayDeque<>();
|
|
22
|
+
List<String> rows = new ArrayList<>();
|
|
23
|
+
Set<Long> entries = new TreeSet<>();
|
|
24
|
+
entries.add(entry.getOffset()); pending.push(entry);
|
|
25
|
+
while (!pending.isEmpty()) {
|
|
26
|
+
if (monitor.isCancelled()) throw new InterruptedException("Cancelled");
|
|
27
|
+
Address at = pending.pop();
|
|
28
|
+
if (visited.contains(at)) continue;
|
|
29
|
+
if (visited.size() >= limit) break; // Missing edge targets remain explicit review gaps.
|
|
30
|
+
visited.add(at);
|
|
31
|
+
Function owner = currentProgram.getFunctionManager().getFunctionContaining(at);
|
|
32
|
+
Function exact = currentProgram.getFunctionManager().getFunctionAt(at);
|
|
33
|
+
if (!at.equals(entry) && exact != null) { entries.add(at.getOffset()); continue; }
|
|
34
|
+
Instruction ins = currentProgram.getListing().getInstructionAt(at);
|
|
35
|
+
if (ins == null) continue;
|
|
36
|
+
List<Long> next = new ArrayList<>(), calls = new ArrayList<>();
|
|
37
|
+
var type = ins.getFlowType();
|
|
38
|
+
Address fall = ins.getFallThrough();
|
|
39
|
+
if (fall != null) { next.add(fall.getOffset()); pending.push(fall); }
|
|
40
|
+
for (Address target : ins.getFlows()) {
|
|
41
|
+
if (type.isCall()) calls.add(target.getOffset());
|
|
42
|
+
else { next.add(target.getOffset()); pending.push(target); }
|
|
43
|
+
}
|
|
44
|
+
String kind = type.isComputed() ? "indirect" : type.isCall() ? "call" : type.isTerminal() ? "terminal" : type.isJump() ? "branch" : "ordinary";
|
|
45
|
+
// A terminal mnemonic alone is insufficient to identify interrupt/OS semantics.
|
|
46
|
+
String mnemonic = ins.getMnemonicString().toUpperCase(Locale.ROOT);
|
|
47
|
+
if (mnemonic.startsWith("RET")) kind = "return";
|
|
48
|
+
boolean hardware = mnemonic.matches("(?:IN|OUT)(?:S[BDW]?)?(?:\\.REP\\w*)?|INT(?:[13O])?|HLT");
|
|
49
|
+
rows.add("{\"start\":" + at.getOffset() + ",\"size\":" + ins.getLength()
|
|
50
|
+
+ ",\"kind\":\"" + kind + "\",\"next\":" + next + ",\"calls\":" + calls
|
|
51
|
+
+ ",\"externalEffects\":" + (hardware ? "[\"instruction requires hardware/OS review\"]" : "[]")
|
|
52
|
+
+ ",\"owner\":" + (owner == null ? "null" : owner.getEntryPoint().getOffset()) + "}");
|
|
53
|
+
}
|
|
54
|
+
Files.createDirectories(output.getParent());
|
|
55
|
+
String text = "{\"entries\":" + entries + ",\"instructions\":[" + String.join(",", rows) + "]}";
|
|
56
|
+
Files.writeString(output, text, StandardCharsets.UTF_8, StandardOpenOption.CREATE_NEW);
|
|
57
|
+
println("Exported " + rows.size() + " instruction metadata records. Ghidra addresses are view-specific; retain the import mapping separately.");
|
|
58
|
+
}
|
|
59
|
+
}
|
scientific_method_engine-0.1.0/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
// Exports normalized function and instruction fingerprints for cross-version mapping.
|
|
2
|
+
// @category Restoration
|
|
3
|
+
|
|
4
|
+
import java.io.BufferedWriter;
|
|
5
|
+
import java.io.File;
|
|
6
|
+
import java.io.FileWriter;
|
|
7
|
+
import java.nio.charset.StandardCharsets;
|
|
8
|
+
import java.security.MessageDigest;
|
|
9
|
+
import java.util.ArrayList;
|
|
10
|
+
import java.util.List;
|
|
11
|
+
|
|
12
|
+
import ghidra.app.script.GhidraScript;
|
|
13
|
+
import ghidra.program.model.address.Address;
|
|
14
|
+
import ghidra.program.model.lang.Register;
|
|
15
|
+
import ghidra.program.model.listing.Function;
|
|
16
|
+
import ghidra.program.model.listing.FunctionIterator;
|
|
17
|
+
import ghidra.program.model.listing.Instruction;
|
|
18
|
+
import ghidra.program.model.listing.InstructionIterator;
|
|
19
|
+
import ghidra.program.model.scalar.Scalar;
|
|
20
|
+
|
|
21
|
+
public class ExportFunctionFingerprints extends GhidraScript {
|
|
22
|
+
@Override
|
|
23
|
+
protected void run() throws Exception {
|
|
24
|
+
String[] args = getScriptArgs();
|
|
25
|
+
if (args.length != 1) {
|
|
26
|
+
throw new IllegalArgumentException("Expected output TSV path.");
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
File output = new File(args[0]);
|
|
30
|
+
File parent = output.getParentFile();
|
|
31
|
+
if (parent != null) {
|
|
32
|
+
parent.mkdirs();
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
try (BufferedWriter writer = new BufferedWriter(new FileWriter(output, StandardCharsets.UTF_8))) {
|
|
36
|
+
writer.write("kind\tfunction\taddress\tend\tsize\tinstructionCount\tfingerprint\tsignature\n");
|
|
37
|
+
FunctionIterator functions = currentProgram.getFunctionManager().getFunctions(true);
|
|
38
|
+
while (functions.hasNext() && !monitor.isCancelled()) {
|
|
39
|
+
Function function = functions.next();
|
|
40
|
+
List<Instruction> instructions = new ArrayList<>();
|
|
41
|
+
InstructionIterator iterator = currentProgram.getListing()
|
|
42
|
+
.getInstructions(function.getBody(), true);
|
|
43
|
+
StringBuilder functionSignature = new StringBuilder();
|
|
44
|
+
while (iterator.hasNext()) {
|
|
45
|
+
Instruction instruction = iterator.next();
|
|
46
|
+
instructions.add(instruction);
|
|
47
|
+
functionSignature.append(normalize(instruction)).append(';');
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
String entry = hex(function.getEntryPoint());
|
|
51
|
+
writer.write(String.join("\t",
|
|
52
|
+
"F", entry, entry, hex(function.getBody().getMaxAddress()),
|
|
53
|
+
Long.toString(function.getBody().getNumAddresses()),
|
|
54
|
+
Integer.toString(instructions.size()), sha256(functionSignature.toString()),
|
|
55
|
+
functionSignature.toString()));
|
|
56
|
+
writer.newLine();
|
|
57
|
+
|
|
58
|
+
for (Instruction instruction : instructions) {
|
|
59
|
+
String signature = normalize(instruction);
|
|
60
|
+
writer.write(String.join("\t",
|
|
61
|
+
"I", entry, hex(instruction.getAddress()),
|
|
62
|
+
hex(instruction.getMaxAddress()), Integer.toString(instruction.getLength()),
|
|
63
|
+
"1", sha256(signature), signature));
|
|
64
|
+
writer.newLine();
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
InstructionIterator allInstructions = currentProgram.getListing().getInstructions(true);
|
|
69
|
+
while (allInstructions.hasNext() && !monitor.isCancelled()) {
|
|
70
|
+
Instruction instruction = allInstructions.next();
|
|
71
|
+
Function function = currentProgram.getFunctionManager()
|
|
72
|
+
.getFunctionContaining(instruction.getAddress());
|
|
73
|
+
String owner = function == null ? "" : hex(function.getEntryPoint());
|
|
74
|
+
String signature = normalize(instruction);
|
|
75
|
+
writer.write(String.join("\t",
|
|
76
|
+
"A", owner, hex(instruction.getAddress()),
|
|
77
|
+
hex(instruction.getMaxAddress()), Integer.toString(instruction.getLength()),
|
|
78
|
+
"1", sha256(signature), signature));
|
|
79
|
+
writer.newLine();
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
println("Wrote " + output.getAbsolutePath());
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// A scalar that names an address in the program is relocatable between versions, so it is
|
|
86
|
+
// fingerprinted as an address rather than by value.
|
|
87
|
+
private boolean isProgramAddress(long value) {
|
|
88
|
+
try {
|
|
89
|
+
return currentProgram.getMemory().contains(toAddr(value));
|
|
90
|
+
} catch (RuntimeException outOfRange) {
|
|
91
|
+
return false;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
private String normalize(Instruction instruction) {
|
|
96
|
+
StringBuilder result = new StringBuilder(instruction.getMnemonicString().toLowerCase());
|
|
97
|
+
for (int operand = 0; operand < instruction.getNumOperands(); operand++) {
|
|
98
|
+
result.append('|');
|
|
99
|
+
Object[] objects = instruction.getOpObjects(operand);
|
|
100
|
+
for (Object object : objects) {
|
|
101
|
+
if (object instanceof Register register) {
|
|
102
|
+
result.append('R').append(register.getName().toLowerCase());
|
|
103
|
+
} else if (object instanceof Address address) {
|
|
104
|
+
result.append(address.isMemoryAddress() ? "A" : "X");
|
|
105
|
+
} else if (object instanceof Scalar scalar) {
|
|
106
|
+
long value = scalar.getSignedValue();
|
|
107
|
+
long unsigned = scalar.getUnsignedValue();
|
|
108
|
+
if (instruction.getFlowType().isFlow() || isProgramAddress(unsigned)) {
|
|
109
|
+
result.append('A');
|
|
110
|
+
} else {
|
|
111
|
+
result.append('S').append(value);
|
|
112
|
+
}
|
|
113
|
+
} else {
|
|
114
|
+
result.append('O').append(object.getClass().getSimpleName());
|
|
115
|
+
}
|
|
116
|
+
result.append(',');
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return result.toString();
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
private static String hex(Address address) {
|
|
123
|
+
return String.format("%08x", address.getOffset());
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
private static String sha256(String value) throws Exception {
|
|
127
|
+
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
|
128
|
+
byte[] hash = digest.digest(value.getBytes(StandardCharsets.UTF_8));
|
|
129
|
+
StringBuilder text = new StringBuilder(hash.length * 2);
|
|
130
|
+
for (byte b : hash) {
|
|
131
|
+
text.append(String.format("%02x", b & 0xff));
|
|
132
|
+
}
|
|
133
|
+
return text.toString();
|
|
134
|
+
}
|
|
135
|
+
}
|