scientific-method-engine 17.0.0__tar.gz → 18.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/PKG-INFO +7 -4
  2. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/README.md +4 -1
  3. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/pyproject.toml +2 -2
  4. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/cli.py +33 -4
  5. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/x86/image.py +24 -3
  6. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/x86/pcode.py +9 -3
  7. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/x86/pcode_backend.py +14 -0
  8. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/x86/relational.py +3 -3
  9. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/x86/values.py +6 -3
  10. scientific_method_engine-18.0.0/tests/test_config_encoding.py +92 -0
  11. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_oracle.py +41 -0
  12. scientific_method_engine-18.0.0/tests/test_upstream_gaps.py +36 -0
  13. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_x86.py +43 -1
  14. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/.gitignore +0 -0
  15. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/LICENSE +0 -0
  16. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/__init__.py +0 -0
  17. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/__main__.py +0 -0
  18. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  19. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  20. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  21. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  22. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  23. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  24. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  25. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  26. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  27. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  28. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  29. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  30. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  31. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  32. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  33. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  34. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  35. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  36. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  37. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  38. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  39. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  40. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  41. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  42. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  43. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  44. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportRangeBoundaries.java +0 -0
  45. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportReferences.java +0 -0
  46. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  47. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  48. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  49. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
  50. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
  51. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
  52. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/ghidra/scientificmethod/Xxh3.java +0 -0
  53. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  54. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
  55. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  56. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  57. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/x86/inventory.py +0 -0
  58. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/x86/loops.py +0 -0
  59. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/x86/machine.py +0 -0
  60. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
  61. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/x86/pe.py +0 -0
  62. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/x86/reach.py +0 -0
  63. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/x86/reports.py +0 -0
  64. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  65. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/src/scientific_method_engine/x86/trace.py +0 -0
  66. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/oracle.py +0 -0
  67. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_continuation_budget.py +0 -0
  68. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_dispatch.py +0 -0
  69. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_effect_order.py +0 -0
  70. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_enter_frames.py +0 -0
  71. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_entry_frame.py +0 -0
  72. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_indirect_far_transfers.py +0 -0
  73. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_interrupt_models.py +0 -0
  74. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_inventory_check.py +0 -0
  75. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_join_terms.py +0 -0
  76. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_loops.py +0 -0
  77. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_memory_scopes.py +0 -0
  78. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_nested_frame_request.py +0 -0
  79. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_pe.py +0 -0
  80. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_reach.py +0 -0
  81. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_relational_controls.py +0 -0
  82. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_return_checks.py +0 -0
  83. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_successor_rule.py +0 -0
  84. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_table_continuations.py +0 -0
  85. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_term_limit.py +0 -0
  86. {scientific_method_engine-17.0.0 → scientific_method_engine-18.0.0}/tests/test_volatile_memory.py +0 -0
@@ -1,14 +1,14 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 17.0.0
3
+ Version: 18.0.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
7
7
  License-Expression: MIT
8
8
  License-File: LICENSE
9
9
  Requires-Python: >=3.12
10
- Requires-Dist: capstone==5.0.7
11
- Requires-Dist: pypcode==4.0.0
10
+ Requires-Dist: capstone==5.0.9
11
+ Requires-Dist: pypcode==4.0.1
12
12
  Requires-Dist: xxhash==4.0.1
13
13
  Provides-Extra: test
14
14
  Requires-Dist: unicorn==2.1.4; extra == 'test'
@@ -29,7 +29,10 @@ For original MZ/FBOV executables, run reports through
29
29
  [`@scientific-method/executable-reader`](https://www.npmjs.com/package/@scientific-method/executable-reader). The
30
30
  reader derives relocation, fixup and trampoline data from the hash-checked source and pipes a
31
31
  prepared config to this engine. Running the engine directly trusts whatever relocation data the
32
- config supplies, so use it directly only for synthetic inputs, PE32 sources and checked mappings:
32
+ config supplies, so use it directly only for synthetic inputs, PE32 sources and checked mappings. A config
33
+ file is read as the reader reads it: UTF-8, with or without one leading byte order mark, and a file starting
34
+ with a UTF-16 or UTF-32 byte order mark or with more than one UTF-8 mark, or bytes that are not UTF-8, are
35
+ refused with the cause named.
33
36
 
34
37
  ```sh
35
38
  scientific-method-engine trace analysis/query.json
@@ -13,7 +13,10 @@ For original MZ/FBOV executables, run reports through
13
13
  [`@scientific-method/executable-reader`](https://www.npmjs.com/package/@scientific-method/executable-reader). The
14
14
  reader derives relocation, fixup and trampoline data from the hash-checked source and pipes a
15
15
  prepared config to this engine. Running the engine directly trusts whatever relocation data the
16
- config supplies, so use it directly only for synthetic inputs, PE32 sources and checked mappings:
16
+ config supplies, so use it directly only for synthetic inputs, PE32 sources and checked mappings. A config
17
+ file is read as the reader reads it: UTF-8, with or without one leading byte order mark, and a file starting
18
+ with a UTF-16 or UTF-32 byte order mark or with more than one UTF-8 mark, or bytes that are not UTF-8, are
19
+ refused with the cause named.
17
20
 
18
21
  ```sh
19
22
  scientific-method-engine trace analysis/query.json
@@ -5,13 +5,13 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "17.0.0"
8
+ version = "18.0.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
12
12
  license = "MIT"
13
13
  authors = [{ name = "kibertoad" }]
14
- dependencies = ["capstone==5.0.7", "pypcode==4.0.0", "xxhash==4.0.1"]
14
+ dependencies = ["capstone==5.0.9", "pypcode==4.0.1", "xxhash==4.0.1"]
15
15
 
16
16
  [project.optional-dependencies]
17
17
  # Unicorn is the tests' concrete oracle (ADR 0003, decision 4). Its core is GPLv2, so it never
@@ -3,15 +3,15 @@ import json
3
3
  import sys
4
4
  from pathlib import Path
5
5
 
6
- import capstone
7
6
  import pypcode
8
7
 
9
8
  from . import PREPARED_PROTOCOL
9
+ from .x86.image import CAPSTONE_VERSION
10
10
 
11
11
  CONFIG_LIMIT = 1024 * 1024
12
12
  PREPARED_CONFIG_LIMIT = 16 * 1024 * 1024
13
13
  # The header names the decoder and instruction semantics that actually ran, not the pins in pyproject.toml.
14
- DECODER = "capstone " + capstone.__version__
14
+ DECODER = f"capstone {CAPSTONE_VERSION or 'without distribution metadata'}"
15
15
  INSTRUCTION_SEMANTICS = f"pypcode {pypcode.__version__} (Ghidra SLEIGH x86)"
16
16
  USAGE = ("Usage: scientific-method-engine <operand|operand-candidates|target|bounds|owner|callees|reach|trace|uses|arguments|"
17
17
  "effects|returns|memory|incoming|inventory-check|call-order|guards|allocation|dispatch> <config.json|->\n"
@@ -52,6 +52,24 @@ def ghidra_scripts():
52
52
  return Path(__file__).resolve().parent / "ghidra"
53
53
 
54
54
 
55
+ def _read_config_text(path):
56
+ # Windows PowerShell 5.1 writes a UTF-8 byte order mark with -Encoding utf8 and UTF-16 by default.
57
+ # The reader's readConfigText in report.ts applies the same rules with the same messages.
58
+ data = path.read_bytes()
59
+ if data[:4] in (b"\xff\xfe\x00\x00", b"\x00\x00\xfe\xff"):
60
+ raise ValueError(f"Config {path} is UTF-32 text; save it as UTF-8")
61
+ if data[:2] in (b"\xff\xfe", b"\xfe\xff"):
62
+ raise ValueError(f"Config {path} is UTF-16 text; save it as UTF-8")
63
+ try:
64
+ # utf-8-sig drops one leading byte order mark.
65
+ text = data.decode("utf-8-sig")
66
+ except UnicodeDecodeError:
67
+ raise ValueError(f"Config {path} is not valid UTF-8; save it as UTF-8") from None
68
+ if text.startswith("\ufeff"):
69
+ raise ValueError(f"Config {path} starts with more than one byte order mark; save it as UTF-8 with at most one")
70
+ return text
71
+
72
+
55
73
  def main(argv):
56
74
  """Run one command (``argv`` excludes the program name) and print its JSON report."""
57
75
  from .x86.image import read_source
@@ -66,14 +84,22 @@ def main(argv):
66
84
  if config_path == "-":
67
85
  # The reader caps its input at 1 MiB, then adds every source relocation.
68
86
  limit, label = PREPARED_CONFIG_LIMIT, "Prepared config exceeds 16 MiB"
69
- text = sys.stdin.read(limit + 1)
87
+ # The reader writes UTF-8. sys.stdin decodes with the locale's encoding (cp1252 on Windows), which
88
+ # would garble a non-ASCII source path or evidence string, so the bytes are decoded here.
89
+ data = sys.stdin.buffer.read(limit + 1)
90
+ if len(data) > limit:
91
+ raise ValueError(label)
92
+ try:
93
+ text = data.decode("utf-8")
94
+ except UnicodeDecodeError:
95
+ raise ValueError("Prepared config is not valid UTF-8") from None
70
96
  base = Path.cwd()
71
97
  else:
72
98
  limit, label = CONFIG_LIMIT, "Config exceeds 1 MiB"
73
99
  path = Path(config_path).resolve()
74
100
  if path.stat().st_size > limit:
75
101
  raise ValueError(label)
76
- text, base = path.read_text(encoding="utf-8"), path.parent
102
+ text, base = _read_config_text(path), path.parent
77
103
  reject_derived = True
78
104
  if len(text.encode("utf-8")) > limit:
79
105
  raise ValueError(label)
@@ -110,6 +136,9 @@ def main(argv):
110
136
 
111
137
  def run():
112
138
  """Entry point of the ``scientific-method-engine`` command; exits with 1 on any error."""
139
+ # The reader decodes this process's stderr as UTF-8; the locale's encoding (cp1252 on Windows) would
140
+ # garble a non-ASCII path in an error message.
141
+ sys.stderr.reconfigure(encoding="utf-8", errors="backslashreplace")
113
142
  try:
114
143
  main(sys.argv[1:])
115
144
  except (ValueError, TypeError, KeyError, OSError, ImportError) as error:
@@ -1,12 +1,30 @@
1
1
  """Bounded explicit code mappings. No guessed linear disassembly domains."""
2
2
  import re
3
+ from importlib import metadata
3
4
  from pathlib import Path
5
+ import pypcode
4
6
  import xxhash
5
7
  from capstone import Cs, CS_ARCH_X86, CS_MODE_16, CS_MODE_32
6
8
  from capstone.x86 import X86_OP_MEM
7
9
  from .pe import prepare_pe
8
- import capstone
9
10
 
11
+
12
+ def installed(name):
13
+ """The installed distribution's version, or None when it has no distribution metadata."""
14
+ try:
15
+ return metadata.version(name)
16
+ except metadata.PackageNotFoundError:
17
+ return None
18
+
19
+
20
+ # The installed Capstone distribution's version. capstone.__version__ cannot stand in for it: the
21
+ # 5.0.8 and 5.0.9 bindings still report 5.0.7, because their CS_VERSION_EXTRA was not bumped.
22
+ CAPSTONE_VERSION = installed("capstone")
23
+ REQUIRED_CAPSTONE = "5.0.9"
24
+ # The engine's rules for which flags to keep are checked against this SLEIGH specification, such as
25
+ # the OF of SHR, which pypcode 4.0.0 writes as 0 where 4.0.1 writes the CPU's value.
26
+ PYPCODE_VERSION = pypcode.__version__
27
+ REQUIRED_PYPCODE = "4.0.1"
10
28
  MAX_SOURCE = 256 * 1024 * 1024
11
29
  XXH3_FORM = re.compile("[0-9a-f]{32}")
12
30
 
@@ -65,8 +83,11 @@ def read_source(config, base):
65
83
 
66
84
  class Image:
67
85
  def __init__(self, data, config):
68
- if capstone.__version__ != "5.0.7":
69
- raise ValueError("This reporter requires capstone==5.0.7")
86
+ for name, required, found in (("capstone", REQUIRED_CAPSTONE, CAPSTONE_VERSION),
87
+ ("pypcode", REQUIRED_PYPCODE, PYPCODE_VERSION)):
88
+ if found != required:
89
+ present = f"{name} {found} is installed" if found else f"the installed {name} has no distribution metadata"
90
+ raise ValueError(f"This reporter requires {name}=={required}; {present}")
70
91
  if config.get("sourceKind") == "pe32":
71
92
  config = prepare_pe(data, config)
72
93
  self.config = config
@@ -315,9 +315,6 @@ def evaluate(code, args, bits, site):
315
315
  and x.term[1][2] + x.term[2][1] == bits):
316
316
  high, low = Value(x.term[1][2], x.term[1][1]), Value(y.term[2], y.term[1])
317
317
  return Value(bits, join([low, high]).term, origin)
318
- if name in ("shl", "shr", "sar") and b.term[0] == "and" and b.term[2] == ("constant", 31):
319
- # x86 masks shift counts to five bits; the engine's shift terms carry that mask already.
320
- b = Value(b.bits, b.term[1], b.sources)
321
318
  if name in ("and", "or", "xor") and not known:
322
319
  # Identities that keep flag expressions small; constants fold in values.op.
323
320
  mask = (1 << bits) - 1
@@ -331,6 +328,15 @@ def evaluate(code, args, bits, site):
331
328
  return Value(bits, y.term, origin)
332
329
  if name == "or" and (x.number == mask):
333
330
  return Value(bits, ("constant", mask), origin)
331
+ if name == "mul" and not known and bits == 8:
332
+ # SLEIGH's conditionalAssign selects a flag as (c * new) | (!c * old); with c known,
333
+ # one arm is a product by zero and the other a product by one. Those products are one
334
+ # byte wide. A wider product is a MUL or IMUL, which keeps its multiply in reports.
335
+ for x, y in ((a, b), (b, a)):
336
+ if x.number == 0:
337
+ return Value(bits, ("constant", 0), origin)
338
+ if x.number == 1:
339
+ return Value(bits, y.term, origin)
334
340
  return op(name, a, b, site)
335
341
  if code == "INT_NEGATE":
336
342
  return op("xor", a, const((1 << bits) - 1, bits), site)
@@ -746,9 +746,21 @@ def shift(state, ins, image):
746
746
  state.carry = carry if carry is not None else unknown(f"carry:{state.at}:{state.flag_serial}", 1, state.at)
747
747
  flags_written(state, f.run)
748
748
  state.flag_values["CF"] = resize(state.carry, 8)
749
+ # SLEIGH's SHR with a count operand (C0, C1, D2, D3) swaps the arms of its OF selection,
750
+ # writing 0 for a count of 1; only the D0 and D1 forms write OF as the CPU does.
751
+ if n != 1 or (m == "shr" and ins.opcode[0] not in (0xD0, 0xD1)):
752
+ undefined_overflow(state)
749
753
  state.event("arithmetic", operation=m, left=a.report(), right=b.report(), result=result.report(), modulus=1 << a.bits)
750
754
 
751
755
 
756
+ def undefined_overflow(state):
757
+ """Drop OF after a shift or rotate whose OF the CPU leaves undefined or SLEIGH writes wrongly.
758
+
759
+ The CPU defines OF only for a count of 1. A branch that reads OF afterwards stays undecided.
760
+ """
761
+ state.flag_values.pop("OF", None)
762
+
763
+
752
764
  def rotate(state, ins, image):
753
765
  m = ins.mnemonic
754
766
  operands = ins.operands
@@ -790,6 +802,8 @@ def rotate(state, ins, image):
790
802
  state.carry = Value(1, carry.term, sources(a, count, site=state.at))
791
803
  flags_written(state, f.run)
792
804
  state.flag_values["CF"] = resize(state.carry, 8)
805
+ if count.number & 31 != 1:
806
+ undefined_overflow(state)
793
807
  state.event("arithmetic", operation=m, left=a.report(), count=n, result=value.report(),
794
808
  carryOut=state.carry.report(), modulus=1 << bits)
795
809
 
@@ -331,9 +331,9 @@ def _unsigned(term, bits, ranges):
331
331
  return 0, top
332
332
  if tag in ("shr", "sar"):
333
333
  # With the sign bit clear, an arithmetic shift right is the logical one.
334
- # Shift terms carry x86's five-bit count mask; a count the mask changes keeps only the upper bound.
335
- s = k & 31
336
- return lo >> s if s == k else 0, hi >> s
334
+ # A p-code shift takes the whole count (values.op): SLEIGH masks an x86 count itself.
335
+ s = min(k, bits)
336
+ return lo >> s, hi >> s
337
337
  if k == 0:
338
338
  return 0, top
339
339
  return (lo // k, hi // k) if tag == "udiv" else (0, min(hi, k - 1))
@@ -183,9 +183,12 @@ def op(name, a, b, site=None):
183
183
  elif name == "and": n = x & y
184
184
  elif name == "or": n = x | y
185
185
  elif name == "xor": n = x ^ y
186
- elif name == "shl": n = x << (y & 31)
187
- elif name == "shr": n = x >> (y & 31)
188
- elif name == "sar": n = (x - (1 << bits) if x & (1 << (bits - 1)) else x) >> (y & 31)
186
+ # P-code shifts take the whole count: SLEIGH masks an x86 count itself, and its 33-bit
187
+ # rotate through CF shifts a 64-bit temporary by 32. A count of the width or more leaves 0,
188
+ # or the sign for an arithmetic shift.
189
+ elif name == "shl": n = x << min(y, bits)
190
+ elif name == "shr": n = x >> min(y, bits)
191
+ elif name == "sar": n = (x - (1 << bits) if x & (1 << (bits - 1)) else x) >> min(y, bits)
189
192
  elif name == "mul": n = x * y
190
193
  else: raise ValueError("Unsupported expression operation: " + name)
191
194
  return Value(bits, const(n, bits).term, origin)
@@ -0,0 +1,92 @@
1
+ """Synthetic machine code and synthetic configs only. No original binaries or analysis artifacts."""
2
+ import json
3
+ import os
4
+ from pathlib import Path
5
+ import subprocess
6
+ import sys
7
+ import tempfile
8
+ import unittest
9
+
10
+ SRC = Path(__file__).resolve().parents[1] / "src"
11
+ import xxhash
12
+
13
+ ENGINE = [sys.executable, "-B", "-m", "scientific_method_engine"]
14
+ ENGINE_ENV = {**os.environ, "PYTHONPATH": os.pathsep.join(filter(None, [str(SRC), os.environ.get("PYTHONPATH")]))}
15
+
16
+ # One resident segment at 1000h: 0000 calls 0004, 0003 returns, 0004 returns.
17
+ CODE = bytes.fromhex("e80100" "c3" "c3")
18
+
19
+
20
+ class ConfigEncodingTests(unittest.TestCase):
21
+ def setUp(self):
22
+ folder = tempfile.TemporaryDirectory()
23
+ self.addCleanup(folder.cleanup)
24
+ self.root = Path(folder.name)
25
+ (self.root / "fixture.bin").write_bytes(CODE)
26
+ # The evidence text holds a non-ASCII letter, so the config's Latin-1 bytes are not UTF-8.
27
+ self.text = json.dumps({
28
+ "source": "fixture.bin", "sourceKind": "synthetic-raw", "xxh3": xxhash.xxh3_128_hexdigest(CODE),
29
+ "regions": [{"name": "resident", "start": 0, "end": len(CODE), "ip": 0, "segment": 0x1000,
30
+ "resident": True, "entries": [0], "evidence": "synthetic code extent, côte"}],
31
+ "target": 4}, ensure_ascii=False)
32
+
33
+ def run_config(self, data):
34
+ path = self.root / "config.json"
35
+ path.write_bytes(data)
36
+ return subprocess.run([*ENGINE, "incoming", str(path)], capture_output=True, text=True, encoding="utf-8",
37
+ env=ENGINE_ENV)
38
+
39
+ def test_a_config_reads_the_same_with_and_without_a_utf8_byte_order_mark(self):
40
+ plain = self.run_config(self.text.encode("utf-8"))
41
+ self.assertEqual(plain.returncode, 0, plain.stderr)
42
+ marked = self.run_config(b"\xef\xbb\xbf" + self.text.encode("utf-8"))
43
+ self.assertEqual(marked.returncode, 0, marked.stderr)
44
+ self.assertEqual(json.loads(marked.stdout), json.loads(plain.stdout))
45
+ self.assertTrue(json.loads(plain.stdout)["confirmed"])
46
+
47
+ def test_a_second_byte_order_mark_is_refused_by_name(self):
48
+ result = self.run_config(b"\xef\xbb\xbf\xef\xbb\xbf" + self.text.encode("utf-8"))
49
+ self.assertEqual(result.returncode, 1)
50
+ self.assertIn("starts with more than one byte order mark", result.stderr)
51
+
52
+ def test_a_prepared_config_on_stdin_is_utf8_whatever_the_locale(self):
53
+ # PYTHONIOENCODING stands in for a Windows locale: sys.stdin would decode the reader's UTF-8 as Latin-1.
54
+ folder = self.root / "côte"
55
+ folder.mkdir()
56
+ (folder / "fixture.bin").write_bytes(CODE)
57
+ prepared = {**json.loads(self.text), "source": str(folder / "fixture.bin"), "preparedProtocol": 3}
58
+ result = subprocess.run([*ENGINE, "incoming", "-"], input=json.dumps(prepared, ensure_ascii=False).encode("utf-8"),
59
+ capture_output=True, env={**ENGINE_ENV, "PYTHONIOENCODING": "latin-1"})
60
+ self.assertEqual(result.returncode, 0, result.stderr)
61
+ self.assertTrue(json.loads(result.stdout)["confirmed"])
62
+ result = subprocess.run([*ENGINE, "incoming", "-"], input=self.text.encode("latin-1"), capture_output=True,
63
+ env=ENGINE_ENV)
64
+ self.assertEqual(result.returncode, 1)
65
+ self.assertIn(b"Prepared config is not valid UTF-8", result.stderr)
66
+
67
+ def test_an_error_naming_a_non_ascii_path_is_written_as_utf8_whatever_the_locale(self):
68
+ # The reader decodes the engine's stderr as UTF-8; PYTHONIOENCODING stands in for a Windows locale.
69
+ folder = self.root / "côte"
70
+ folder.mkdir()
71
+ (folder / "config.json").write_bytes(b"\xff\xfe" + self.text.encode("utf-16-le"))
72
+ result = subprocess.run([*ENGINE, "incoming", str(folder / "config.json")], capture_output=True,
73
+ env={**ENGINE_ENV, "PYTHONIOENCODING": "latin-1"})
74
+ self.assertEqual(result.returncode, 1)
75
+ self.assertIn(f"Config {folder / 'config.json'} is UTF-16 text", result.stderr.decode("utf-8"))
76
+
77
+ def test_utf16_utf32_and_invalid_utf8_configs_are_refused_with_the_encoding_named(self):
78
+ for data in (b"\xff\xfe" + self.text.encode("utf-16-le"), b"\xfe\xff" + self.text.encode("utf-16-be")):
79
+ result = self.run_config(data)
80
+ self.assertEqual(result.returncode, 1)
81
+ self.assertIn("is UTF-16 text; save it as UTF-8", result.stderr)
82
+ for data in (b"\xff\xfe\x00\x00" + self.text.encode("utf-32-le"), b"\x00\x00\xfe\xff" + self.text.encode("utf-32-be")):
83
+ result = self.run_config(data)
84
+ self.assertEqual(result.returncode, 1)
85
+ self.assertIn("is UTF-32 text; save it as UTF-8", result.stderr)
86
+ result = self.run_config(self.text.encode("latin-1"))
87
+ self.assertEqual(result.returncode, 1)
88
+ self.assertIn("is not valid UTF-8; save it as UTF-8", result.stderr)
89
+
90
+
91
+ if __name__ == "__main__":
92
+ unittest.main()
@@ -187,6 +187,39 @@ class ShiftsAndRotates(unittest.TestCase):
187
187
  result = check(self, "b90100 31d2 d1e0 d1d2 d1e0 d1d2 bb0000 11db c3", resolved=("cx", "bx"))
188
188
  self.assertEqual(result["paths"][0]["registers"]["dx"]["expression"][0], "or")
189
189
 
190
+ def test_overflow_of_one_bit_shifts_and_rotates(self):
191
+ # The engine keeps OF after every shift and rotate by a count of 1 except SHR by a count
192
+ # operand, so each such encoding gets its own routine: JNO skips INC BX when OF is clear.
193
+ # The operands set and clear the top two bits and bit 0, and RCL and RCR run from both carries.
194
+ # 8-bit forms work on AL, 16-bit forms on AX and 32-bit forms (66h) on EAX; CL holds 1.
195
+ forms = {"d0": ("", 8), "d1": ("", 16), "66d1": ("", 32), "c0": ("01", 8), "c1": ("01", 16),
196
+ "66c1": ("01", 32), "d2": ("", 8), "d3": ("", 16), "66d3": ("", 32)}
197
+ for field, mnemonic in enumerate(("rol", "ror", "rcl", "rcr", "shl", "shr", "sal", "sar")):
198
+ for opcode, (immediate, bits) in forms.items():
199
+ if mnemonic == "shr" and opcode[-2:] not in ("d0", "d1"):
200
+ continue
201
+ for top in (0b00, 0b01, 0b10, 0b11):
202
+ load = "66b8" if bits == 32 else "b8"
203
+ value = (top << (bits - 2) | (top & 1)).to_bytes(4 if bits == 32 else 2, "little").hex()
204
+ for carry in (("f8", "f9") if mnemonic in ("rcl", "rcr") else ("",)):
205
+ shift = f"{opcode}{0xC0 | field << 3:02x}{immediate}"
206
+ with self.subTest(mnemonic=mnemonic, shift=shift, value=value, carry=carry):
207
+ check(self, f"bb0000 b90100 {load}{value} {carry} {shift} 7101 43 c3", resolved=("bx",))
208
+
209
+ def test_shift_by_an_unknown_count_keeps_the_count_mask(self):
210
+ # A p-code shift takes its whole count, so the term keeps SLEIGH's five-bit mask of CL.
211
+ result = check(self, "bb0300 d3e0 c3", resolved=("bx",))
212
+ ax = result["paths"][0]["registers"]["ax"]["expression"]
213
+ self.assertEqual((ax[0], ax[2][0], ax[2][1][0], ax[2][1][2]), ("shl", "zeroExtend", "and", ("constant", 31)))
214
+
215
+ def test_double_word_rotates_through_carry(self):
216
+ # SLEIGH rotates EAX through CF in a 64-bit temporary built with zext(CF) << 32, a p-code
217
+ # shift by 32 that x86's five-bit count mask does not apply to.
218
+ for rotate in ("66d1d0", "66d1d8", "66c1d005", "66c1d81f", "66d3d0", "66d3d8"):
219
+ for carry in ("f8", "f9"):
220
+ with self.subTest(rotate=rotate, carry=carry):
221
+ check(self, f"b90300 66b8f00f0180 {carry} {rotate} bb0000 83d300 c3", resolved=("eax", "bx"))
222
+
190
223
  def test_memory_operands(self):
191
224
  # RCL by one on memory: Capstone reports its implicit count with a size of 0.
192
225
  check(self, "bb1000 c7070180 d107 c12f04 f9 d117 8b07 c7070180 f8 d017 8b07 c3", registers=DATA, resolved=("ax",))
@@ -203,6 +236,14 @@ class MultiplyAndDivide(unittest.TestCase):
203
236
  self.assertEqual(registers["cx"]["expression"][0], "mul")
204
237
  self.assertEqual(registers["dx"]["expression"][0], "mul")
205
238
 
239
+ def test_product_by_zero_or_one_keeps_the_multiply(self):
240
+ # BX is unknown; the engine folds only the one-byte products of SLEIGH's flag selections.
241
+ result = check(self, "b90300 6bc301 66 6bd301 6bf300 c3", resolved=("cx",))
242
+ registers = result["paths"][0]["registers"]
243
+ self.assertEqual(registers["ax"]["expression"][0], "mul")
244
+ self.assertEqual(registers["dx"]["expression"][1][0], "mul")
245
+ self.assertEqual(registers["si"]["expression"][0], "mul")
246
+
206
247
  def test_low_products_of_extended_operands(self):
207
248
  # AX holds the sign extension of an unknown byte; the products name AX, not the byte.
208
249
  result = check(self, "a00000 98 0fafc8 6bd0fd b80100 c3", registers=DATA, resolved=("ax",))
@@ -0,0 +1,36 @@
1
+ """Canaries for the upstream gaps the engine works around (upstream-gaps.md). Synthetic bytes only.
2
+
3
+ Each test lifts with the pinned pypcode and asserts the gap is still there. A pypcode that fixes
4
+ one fails its test, which is the signal to drop the workaround instead of letting it match nothing.
5
+ """
6
+ import sys
7
+ import unittest
8
+ from pathlib import Path
9
+
10
+ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
11
+ from scientific_method_engine.x86.pcode import Lifter, cs_idiom # noqa: E402
12
+
13
+ CS_GAP = ("upstream-gaps.md, 'Real mode: a CS override writes CS from the instruction address': the "
14
+ "pinned pypcode no longer emits the sequence pcode.cs_idiom drops. If it carries ghidra#9703, "
15
+ "drop cs_idiom, keep CS from the declared region, and check that the pspec's tracked CS does "
16
+ "not reach reports.")
17
+
18
+
19
+ class RealModeCsOverride(unittest.TestCase):
20
+ def test_cs_override_still_writes_cs_from_the_instruction_address(self):
21
+ lifter = Lifter()
22
+ # mov ax,cs:[bx]; jmp cs:[bx+0030]
23
+ for code in ("2e8b07", "2effa73000"):
24
+ with self.subTest(code=code):
25
+ ops, _ = lifter.ops(False, bytes.fromhex(code), 0x10000)
26
+ skip = sorted(cs_idiom(ops))
27
+ # One sequence per CS-relative access; the indirect JMP's lift has two.
28
+ self.assertTrue(skip and len(skip) % 3 == 0, CS_GAP)
29
+ for start in skip[::3]:
30
+ shift, mask, copy = ops[start:start + 3]
31
+ self.assertEqual((shift.code, mask.code, copy.code), ("INT_RIGHT", "INT_AND", "COPY"), CS_GAP)
32
+ self.assertEqual(lifter.register(False, copy.output[1], copy.output[2]), "CS", CS_GAP)
33
+
34
+
35
+ if __name__ == "__main__":
36
+ unittest.main()
@@ -18,7 +18,9 @@ sys.path.insert(0, str(SRC))
18
18
  ENGINE = [sys.executable, "-B", "-m", "scientific_method_engine"]
19
19
  ENGINE_ENV = {**os.environ, "PYTHONPATH": os.pathsep.join(filter(None, [str(SRC), os.environ.get("PYTHONPATH")]))}
20
20
  import capstone
21
+ from importlib import metadata
21
22
  import pypcode
23
+ from scientific_method_engine.x86 import image as image_module
22
24
  from scientific_method_engine.x86.image import Image
23
25
  from scientific_method_engine.x86 import reports
24
26
  from scientific_method_engine.x86.reports import run_report
@@ -2292,6 +2294,22 @@ class ReporterTests(unittest.TestCase):
2292
2294
  unknown_divisor = report("f7 f3 c3")
2293
2295
  self.assertEqual(unknown_divisor["paths"][0]["conditionalModels"][0]["assumption"], "no divide error")
2294
2296
 
2297
+ def test_overflow_after_a_shift_or_rotate_is_read_only_for_a_count_of_one(self):
2298
+ # CMP AX,AX clears OF first, so a branch that read a stale OF would be decided.
2299
+ def branches(shift):
2300
+ return events(report(f"b8 00 40 b1 01 39 c0 {shift} 70 01 43 c3"), "branch")
2301
+ for shift in ("d1 e0", "d1 e8", "d1 f8", "d1 c0", "d1 c8", "d1 d0", "d1 d8", "d0 e8",
2302
+ "c1 e0 01", "c1 f8 01", "c1 c0 01", "c1 c8 01", "c1 d0 01", "c1 d8 01", "d3 e0", "d3 d8"):
2303
+ with self.subTest(shift=shift):
2304
+ branch, = branches(shift)
2305
+ self.assertEqual(branch["decidedBy"], "p-code flags")
2306
+ # The CPU leaves OF undefined past a count of 1. SLEIGH's SHR with a count operand writes OF
2307
+ # as 0 for a count of 1, where the CPU writes the operand's top bit.
2308
+ for shift in ("c1 e0 03", "c1 e8 02", "c1 f8 02", "c1 c0 03", "c1 c8 03", "c1 d0 03", "c1 d8 03",
2309
+ "c1 e8 01", "d3 e8", "c0 e8 01", "d2 e8", "66 c1 e8 01"):
2310
+ with self.subTest(shift=shift):
2311
+ self.assertEqual([b.get("decidedBy") for b in branches(shift)], [None, None])
2312
+
2295
2313
  def test_carry_review_regressions(self):
2296
2314
  # A wide rotate of an unknown value stays a bounded expression.
2297
2315
  self.assertIsNone(report("c1 c0 08 c3")["paths"][0]["stop"])
@@ -2958,6 +2976,30 @@ class ReporterTests(unittest.TestCase):
2958
2976
  result = subprocess.run([sys.executable, "-B", "-c", code], capture_output=True, text=True, env=ENGINE_ENV)
2959
2977
  self.assertEqual(result.returncode, 0, result.stderr)
2960
2978
 
2979
+ def test_decoder_identity_is_the_installed_capstone_distribution(self):
2980
+ # The Capstone 5.0.8 and 5.0.9 bindings report 5.0.7 as capstone.__version__, so the guard and
2981
+ # the report header read the distribution's own version.
2982
+ self.assertEqual(image_module.CAPSTONE_VERSION, metadata.version("capstone"))
2983
+ self.assertEqual(image_module.CAPSTONE_VERSION, image_module.REQUIRED_CAPSTONE)
2984
+ data = bytes.fromhex("c3")
2985
+ with mock.patch.object(image_module, "CAPSTONE_VERSION", "5.0.7"):
2986
+ with self.assertRaisesRegex(ValueError, r"requires capstone==5\.0\.9; capstone 5\.0\.7 is installed"):
2987
+ Image(data, configuration(data))
2988
+ with mock.patch.object(image_module, "CAPSTONE_VERSION", None):
2989
+ with self.assertRaisesRegex(ValueError, r"requires capstone==5\.0\.9; the installed capstone has no distribution metadata"):
2990
+ Image(data, configuration(data))
2991
+ with mock.patch.object(image_module.metadata, "version", side_effect=metadata.PackageNotFoundError("capstone")):
2992
+ self.assertIsNone(image_module.installed("capstone"))
2993
+
2994
+ def test_pypcode_guard_refuses_another_sleigh_specification(self):
2995
+ # Which flags the engine keeps after a shift is checked against pypcode 4.0.1's SLEIGH files;
2996
+ # pypcode 4.0.0 writes the OF of SHR by one as 0.
2997
+ self.assertEqual(image_module.PYPCODE_VERSION, image_module.REQUIRED_PYPCODE)
2998
+ data = bytes.fromhex("c3")
2999
+ with mock.patch.object(image_module, "PYPCODE_VERSION", "4.0.0"):
3000
+ with self.assertRaisesRegex(ValueError, r"requires pypcode==4\.0\.1; pypcode 4\.0\.0 is installed"):
3001
+ Image(data, configuration(data))
3002
+
2961
3003
  def test_cli_identity_and_errors(self):
2962
3004
  with tempfile.TemporaryDirectory() as folder:
2963
3005
  root = Path(folder); data = bytes.fromhex("b8 01 00 c3")
@@ -2970,7 +3012,7 @@ class ReporterTests(unittest.TestCase):
2970
3012
  header = json.loads(result.stdout)
2971
3013
  self.assertEqual(header["sourceIdentity"], {"size": 4, "xxh3": cfg["xxh3"]})
2972
3014
  self.assertEqual((header["decoder"], header["instructionSemantics"]),
2973
- ("capstone " + capstone.__version__, f"pypcode {pypcode.__version__} (Ghidra SLEIGH x86)"))
3015
+ ("capstone " + metadata.version("capstone"), f"pypcode {pypcode.__version__} (Ghidra SLEIGH x86)"))
2974
3016
  cfg["xxh3"] = "0" * 32; path.write_text(json.dumps(cfg))
2975
3017
  result = subprocess.run(args, capture_output=True, text=True, env=ENGINE_ENV)
2976
3018
  self.assertEqual(result.returncode, 1)