scientific-method-engine 13.3.0__tar.gz → 13.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/PKG-INFO +2 -2
  2. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/README.md +1 -1
  3. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/pyproject.toml +1 -1
  4. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportReferences.java +4 -1
  5. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/x86/machine.py +6 -1
  6. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/x86/relational.py +3 -0
  7. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/x86/reports.py +5 -1
  8. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/x86/trace.py +35 -11
  9. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/x86/values.py +19 -5
  10. scientific_method_engine-13.4.0/tests/test_term_limit.py +237 -0
  11. scientific_method_engine-13.3.0/tests/test_term_limit.py +0 -120
  12. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/.gitignore +0 -0
  13. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/LICENSE +0 -0
  14. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/__init__.py +0 -0
  15. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/__main__.py +0 -0
  16. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/cli.py +0 -0
  17. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ClearNoReturnFunctions.java +0 -0
  18. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/CreateFunctions.java +0 -0
  19. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ExportBoundedFlow.java +0 -0
  20. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ExportCallEdges.java +0 -0
  21. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ExportFunctionFingerprints.java +0 -0
  22. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ExportFunctionInventory.java +0 -0
  23. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/MergeFallThroughFragment.java +0 -0
  24. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/RecoverCitedFunctions.java +0 -0
  25. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/RepairReturningCallers.java +0 -0
  26. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportCallArguments.java +0 -0
  27. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportCallPaths.java +0 -0
  28. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportCallSitesWithScalars.java +0 -0
  29. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportCallsToRange.java +0 -0
  30. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportConstantFirstArgumentCalls.java +0 -0
  31. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportDataBytes.java +0 -0
  32. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportDecompileMatches.java +0 -0
  33. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportDecompileWindow.java +0 -0
  34. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportFilePatternInMemory.java +0 -0
  35. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportFirstArgumentCallSummary.java +0 -0
  36. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportFunctionScalarConstants.java +0 -0
  37. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportFunctionSummary.java +0 -0
  38. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportInstructionContext.java +0 -0
  39. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportInstructionWindow.java +0 -0
  40. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportMemoryBlockForFileOffset.java +0 -0
  41. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportMemoryBlocks.java +0 -0
  42. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportRandomnessCandidates.java +0 -0
  43. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportScalarConstants.java +0 -0
  44. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportStringReferences.java +0 -0
  45. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/ReportSymbolReferences.java +0 -0
  46. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/scientificmethod/FirstArgumentLookBack.java +0 -0
  47. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/scientificmethod/InstructionStart.java +0 -0
  48. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/ghidra/scientificmethod/OperandConstants.java +0 -0
  49. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/x86/__init__.py +0 -0
  50. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/x86/argument_frames.py +0 -0
  51. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/x86/dispatch.py +0 -0
  52. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/x86/effect_order.py +0 -0
  53. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/x86/image.py +0 -0
  54. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/x86/loops.py +0 -0
  55. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/x86/memory_scopes.py +0 -0
  56. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/x86/pcode.py +0 -0
  57. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/x86/pcode_backend.py +0 -0
  58. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/x86/pe.py +0 -0
  59. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/x86/reach.py +0 -0
  60. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/src/scientific_method_engine/x86/result_flow.py +0 -0
  61. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/tests/oracle.py +0 -0
  62. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/tests/test_continuation_budget.py +0 -0
  63. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/tests/test_dispatch.py +0 -0
  64. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/tests/test_effect_order.py +0 -0
  65. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/tests/test_enter_frames.py +0 -0
  66. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/tests/test_entry_frame.py +0 -0
  67. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/tests/test_indirect_far_transfers.py +0 -0
  68. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/tests/test_interrupt_models.py +0 -0
  69. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/tests/test_loops.py +0 -0
  70. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/tests/test_memory_scopes.py +0 -0
  71. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/tests/test_nested_frame_request.py +0 -0
  72. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/tests/test_oracle.py +0 -0
  73. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/tests/test_pe.py +0 -0
  74. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/tests/test_reach.py +0 -0
  75. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/tests/test_relational_controls.py +0 -0
  76. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/tests/test_return_checks.py +0 -0
  77. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/tests/test_successor_rule.py +0 -0
  78. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/tests/test_table_continuations.py +0 -0
  79. {scientific_method_engine-13.3.0 → scientific_method_engine-13.4.0}/tests/test_x86.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: scientific-method-engine
3
- Version: 13.3.0
3
+ Version: 13.4.0
4
4
  Summary: Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code.
5
5
  Project-URL: Source, https://github.com/kibertoad/refurbished-dinosaurs-toolkit/tree/main/packages/scientific-method-engine
6
6
  Author: kibertoad
@@ -73,7 +73,7 @@ Finding references and calls:
73
73
 
74
74
  | Script | Arguments | Prints |
75
75
  |---|---|---|
76
- | `ReportReferences` | one or more addresses | references to each, with the referring instruction, function and Ghidra reference type. The type is not the access: `DATA` covers indexed reads and writes and address formation, and a memory operand Ghidra gave no reference is not listed |
76
+ | `ReportReferences` | one or more addresses | references to each, with the referring instruction, function and Ghidra reference type. The type is not the access: `DATA` covers indexed reads and writes and address formation, and a memory operand Ghidra gave no reference is not listed, nor are bytes Ghidra neither disassembled nor defined as a pointer |
77
77
  | `ReportStringReferences` | one or more literal string fragments | strings containing a fragment and their references |
78
78
  | `ReportSymbolReferences` | one or more symbol-name fragments, matched as case-insensitive substrings of the full name | matching symbols, default labels included, and their references. Default labels end in their address, so an address fragment such as `0089d4a4` finds the `PTR_<name>_0089d4a4` import slot there |
79
79
  | `ReportScalarConstants` | optional operand kind (`immediate` or `memory`), one or more scalar values | instructions using any of them, unsigned or signed, as an immediate or inside a memory operand (a displacement such as `[ECX + 0x44]`, an absolute address such as `[0x41c000]`, or an index scale), with the kind on each line |
@@ -57,7 +57,7 @@ Finding references and calls:
57
57
 
58
58
  | Script | Arguments | Prints |
59
59
  |---|---|---|
60
- | `ReportReferences` | one or more addresses | references to each, with the referring instruction, function and Ghidra reference type. The type is not the access: `DATA` covers indexed reads and writes and address formation, and a memory operand Ghidra gave no reference is not listed |
60
+ | `ReportReferences` | one or more addresses | references to each, with the referring instruction, function and Ghidra reference type. The type is not the access: `DATA` covers indexed reads and writes and address formation, and a memory operand Ghidra gave no reference is not listed, nor are bytes Ghidra neither disassembled nor defined as a pointer |
61
61
  | `ReportStringReferences` | one or more literal string fragments | strings containing a fragment and their references |
62
62
  | `ReportSymbolReferences` | one or more symbol-name fragments, matched as case-insensitive substrings of the full name | matching symbols, default labels included, and their references. Default labels end in their address, so an address fragment such as `0089d4a4` finds the `PTR_<name>_0089d4a4` import slot there |
63
63
  | `ReportScalarConstants` | optional operand kind (`immediate` or `memory`), one or more scalar values | instructions using any of them, unsigned or signed, as an immediate or inside a memory operand (a displacement such as `[ECX + 0x44]`, an absolute address such as `[0x41c000]`, or an index scale), with the kind on each line |
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  name = "scientific-method-engine"
7
7
  # The release workflow writes the published version from the package's release tag.
8
- version = "13.3.0"
8
+ version = "13.4.0"
9
9
  description = "Bounded instruction-derived x86 evidence reports for segmented MZ/FBOV and PE32/i386 code."
10
10
  readme = "README.md"
11
11
  requires-python = ">=3.12"
@@ -42,7 +42,10 @@ public class ReportReferences extends GhidraScript {
42
42
 
43
43
  // Ghidra types an indexed store such as MOV [EAX*4+table],ECX as DATA, and gives an indexed
44
44
  // read-modify-write no reference at all, so the type column cannot answer "who writes this".
45
+ // References start only at instructions and defined data, so a value in bytes Ghidra neither
46
+ // disassembled nor defined as a pointer is invisible here.
45
47
  println("Types are Ghidra's reference types, not the access an instruction makes: DATA covers"
46
- + " indexed reads and writes and address formation. An operand Ghidra gave no reference is not listed.");
48
+ + " indexed reads and writes and address formation. An operand Ghidra gave no reference is not listed,"
49
+ + " nor are bytes Ghidra neither disassembled nor defined as a pointer.");
47
50
  }
48
51
  }
@@ -354,7 +354,12 @@ class State:
354
354
  def reg(self, name):
355
355
  root, low, bits = alias(name)
356
356
  value = extract(self.regs[root], low, bits)
357
- return Value(bits, value.term, tuple(sorted(set().union(*self.reg_sources[root][low // 8:(low + bits) // 8]))))
357
+ return Value(bits, value.term, self.reg_origin(name))
358
+
359
+ def reg_origin(self, name):
360
+ """The sources of register ``name``'s bytes, which ``reg(name)`` carries, without forming its value."""
361
+ root, low, bits = alias(name)
362
+ return tuple(sorted(set().union(*self.reg_sources[root][low // 8:(low + bits) // 8])))
358
363
 
359
364
  def setreg(self, name, value, site):
360
365
  root, low, bits = alias(name)
@@ -432,6 +432,9 @@ def _field(event, path):
432
432
  raise _Unresolved(f"event {event['kind']} at {event['site']} has no field {path}")
433
433
  if not isinstance(value, dict) or "expression" not in value or "bits" not in value:
434
434
  raise _Unresolved(f"field {path} of event {event['kind']} at {event['site']} is not a value")
435
+ if value["expression"] is None:
436
+ # A register row the snapshot could not form names why; it has no expression to compare.
437
+ raise _Unresolved(f"field {path} of event {event['kind']} at {event['site']} was not formed: {value['unresolved']}")
435
438
  return value
436
439
 
437
440
 
@@ -643,7 +643,8 @@ def dispatch(image, config):
643
643
  outcomes = []
644
644
  for path in report["paths"]:
645
645
  reached = bool(path["instructionPath"]) and path["instructionPath"][-1] == site
646
- index_value = path["registers"].get(index_reg, {}).get("value")
646
+ index_row = path["registers"].get(index_reg, {})
647
+ index_value = index_row.get("value")
647
648
  if reached and index_value is not None:
648
649
  index_value *= scale
649
650
  if index_value % divisor or index_value // divisor >= count:
@@ -653,6 +654,9 @@ def dispatch(image, config):
653
654
  outcomes.append({"status": "selected", "position": index, "rawTarget": rows[index], "encodedIndex": index_value})
654
655
  else:
655
656
  outcomes.append({"status": "returned-before-dispatch" if path["returned"] else "unresolved", "stop": path["stop"]})
657
+ if reached and "unresolved" in index_row:
658
+ # The site was reached but the index register could not be formed, which the stop alone does not say.
659
+ outcomes[-1]["unresolved"] = index_row["unresolved"]
656
660
  outcomes[-1]["transformations"] = [e for e in path["events"] if e["kind"] == "arithmetic"]
657
661
  outcomes[-1]["guards"] = path["guards"]
658
662
  results.append({"input": value, "outcomes": outcomes, "gaps": report["gaps"]})
@@ -6,7 +6,7 @@ from .image import integer
6
6
  from .machine import (State, StopPath, REGISTERS, ALIASES, BRANCH_CONDITIONS, string_instruction,
7
7
  string_count, string_effect, check_string_form, compare_string, repeated, string_width,
8
8
  FLAT_PORT_REASON)
9
- from .values import const, unknown, sources, op, join, TermLimit
9
+ from .values import const, unknown, sources, op, join, unformed, TermLimit
10
10
  from .result_flow import validate_contracts, result_contracts
11
11
  from .loops import LoopTracker
12
12
  from .memory_scopes import validate_scopes, capture_scopes, retain_scopes, scope_history
@@ -407,8 +407,18 @@ def hardware_placement(outputs, conditional_outputs, gaps):
407
407
  return result
408
408
 
409
409
 
410
+ def register_row(state, name):
411
+ """Register ``name``'s report row. A snapshot only observes the path, so a partial register that
412
+ would pass the term limit (its root holds the limit, and selecting the part adds a term) is a row
413
+ with no expression or value and the limit as its ``unresolved`` reason."""
414
+ try:
415
+ return state.reg(name).report()
416
+ except TermLimit as error:
417
+ return unformed(ALIASES[name][2], state.reg_origin(name), str(error))
418
+
419
+
410
420
  def snapshot(state):
411
- return {name: state.reg(name).report() for name in ALIASES}
421
+ return {name: register_row(state, name) for name in ALIASES}
412
422
 
413
423
 
414
424
  # The stop for each failed combination of (widthMatches, stackBalanced) at a return.
@@ -545,18 +555,25 @@ def entry_frame(image, config, entry):
545
555
  if not arrivals:
546
556
  reasons.append(f"no path from {start} reached the entry")
547
557
  offsets = sorted({a["sp"] for a in arrivals if a["sp"] is not None})
548
- if any(a["sp"] is None for a in arrivals):
558
+ unformed_sp = sorted({a["unresolved"]["sp"] for a in arrivals if "sp" in a.get("unresolved", {})})
559
+ reasons += ["SP at an arrival could not be formed: " + reason for reason in unformed_sp]
560
+ if any(a["sp"] is None and "sp" not in a.get("unresolved", {}) for a in arrivals):
549
561
  reasons.append("SP at an arrival is not an offset from the function's entry SP")
550
562
  elif len(offsets) > 1:
551
563
  reasons.append("arrivals reach the entry with SP at different offsets: " + ", ".join(map(str, offsets)))
552
564
  established = not reasons
553
565
  bp = {a["bp"] for a in arrivals}
554
- return {"from": start, "established": established, "sp": offsets[0] if established else None,
555
- "bp": next(iter(bp)) if established and len(bp) == 1 else None,
556
- "arrivals": len(arrivals), "pathsRead": len(report["paths"]), "stepsUsed": report["stepsUsed"],
557
- "reasons": reasons,
558
- "meaning": "offsets from the entry SP of the function at from, observed at each first arrival on the "
559
- "traced paths under the query's own inputs; bp null leaves BP unknown"}
566
+ frame = {"from": start, "established": established, "sp": offsets[0] if established else None,
567
+ "bp": next(iter(bp)) if established and len(bp) == 1 else None,
568
+ "arrivals": len(arrivals), "pathsRead": len(report["paths"]), "stepsUsed": report["stepsUsed"],
569
+ "reasons": reasons,
570
+ "meaning": "offsets from the entry SP of the function at from, observed at each first arrival on the "
571
+ "traced paths under the query's own inputs; bp null leaves BP unknown"}
572
+ # BP does not decide whether the frame is established, so why it could not be formed has its own field.
573
+ unformed_bp = sorted({a["unresolved"]["bp"] for a in arrivals if "bp" in a.get("unresolved", {})})
574
+ if unformed_bp:
575
+ frame["bpUnresolved"] = unformed_bp
576
+ return frame
560
577
 
561
578
 
562
579
  def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=None, call_stacks=False, argument_window=0):
@@ -730,8 +747,15 @@ def trace(image, config, continue_declared_jumps=True, track_loops=True, arrive=
730
747
  "instructionPath": s.path, "guards": s.guards, "events": s.events,
731
748
  "registers": snapshot(s), "conditionalModels": s.conditional}
732
749
  if reason == ENTRY_ARRIVAL:
733
- path["arrival"] = {"frameEntry": s.frames[-1]["entry"], "sp": s.frame_offset(s.reg(s.sp)),
734
- "bp": s.frame_offset(s.reg(s.bp))}
750
+ arrival = path["arrival"] = {"frameEntry": s.frames[-1]["entry"]}
751
+ for key, name in (("sp", s.sp), ("bp", s.bp)):
752
+ # Finishing a path only observes it, so a register past the term limit is an offset
753
+ # not known, with the limit as the reason.
754
+ try:
755
+ arrival[key] = s.frame_offset(s.reg(name))
756
+ except TermLimit as error:
757
+ arrival[key] = None
758
+ arrival.setdefault("unresolved", {})[key] = str(error)
735
759
  if s.loops is not None:
736
760
  path["loops"] = s.loops.report()
737
761
  if call_stacks and not returned:
@@ -40,11 +40,25 @@ class Value:
40
40
  return self.term[1] if self.term[0] == "constant" else None
41
41
 
42
42
  def report(self):
43
- row = {"bits": self.bits, "expression": self.term, "value": self.number, "producers": producers(self)}
44
- origins = result_origins(self)
45
- if origins:
46
- row["resultOrigins"] = origins
47
- return row
43
+ return _row(self.bits, self.term, self.number, self.sources)
44
+
45
+
46
+ def unformed(bits, origin, reason):
47
+ """The report row of a value the model could not form, with the sources it would have carried.
48
+
49
+ ``expression`` and ``value`` are None, ``producers`` and ``resultOrigins`` come from ``origin``
50
+ as ``Value.report`` gives them, and ``unresolved`` says why the value was not formed.
51
+ """
52
+ return {**_row(bits, None, None, origin), "unresolved": reason}
53
+
54
+
55
+ def _row(bits, expression, value, origin):
56
+ """A report row with the ``producers`` and ``resultOrigins`` that the sources ``origin`` give."""
57
+ row = {"bits": bits, "expression": expression, "value": value, "producers": [s for s in origin if s >= 0]}
58
+ origins = sorted(-s - 1 for s in origin if s < 0)
59
+ if origins:
60
+ row["resultOrigins"] = origins
61
+ return row
48
62
 
49
63
 
50
64
  def result_marker(order):
@@ -0,0 +1,237 @@
1
+ """A value past the term limit stops its own path; the other paths and their controls stay."""
2
+ import copy
3
+ import pickle
4
+ import unittest
5
+ from test_x86 import Code, report
6
+ from scientific_method_engine.x86.relational import probe_memory
7
+ from scientific_method_engine.x86.values import TERM_LIMIT, TermLimit, Value, const, op
8
+
9
+ FRAME = {"ds": 0x3000, "ss": 0x9000, "sp": 0xE000}
10
+ STOP = f"expression term limit: a value's expression would hold more than {TERM_LIMIT} terms; narrow the query"
11
+
12
+
13
+ def additions(count):
14
+ """AX and BX load unknown words; a zero AX skips ``count`` additions of BX into AX."""
15
+ c = Code().emit("a1 00 01 8b 1e 02 01").label("test").emit("85 c0 0f 84")
16
+ c.fixups.append((len(c.data), 2, "done"))
17
+ c.data.extend(bytes(2))
18
+ c.label("adds")
19
+ for _ in range(count):
20
+ c.emit("01 d8")
21
+ return c.label("done").emit("cb")
22
+
23
+
24
+ def run(count, **extra):
25
+ return report(additions(count), "trace", **{"returnBytes": 4, "registers": FRAME, "maxPaths": 2, "maxSteps": count + 16, **extra})
26
+
27
+
28
+ def negations(count):
29
+ """EAX loads an unknown dword and BX a constant; ``count`` NOTs of EAX follow, the last at ``last``."""
30
+ c = Code().emit("66 a1 00 01 bb 34 12")
31
+ for index in range(count):
32
+ if index == count - 1:
33
+ c.label("last")
34
+ c.emit("66 f7 d0")
35
+ return c.label("done").emit("cb")
36
+
37
+
38
+ def negate(count, **extra):
39
+ c = negations(count)
40
+ return report(c, "trace", **{"returnBytes": 4, "registers": FRAME, "maxSteps": count + 16,
41
+ "checkpoints": [c.labels["done"]], **extra})
42
+
43
+
44
+ class TermLimitTests(unittest.TestCase):
45
+ def test_a_short_sum_returns_on_both_paths(self):
46
+ r = run(8)
47
+ self.assertEqual([p["returned"] for p in r["paths"]], [True, True])
48
+ self.assertTrue(r["completeWithinModel"])
49
+
50
+ def test_a_value_past_the_limit_stops_its_path_and_keeps_the_other(self):
51
+ c = additions(600)
52
+ r = run(600)
53
+ returned, stopped = r["paths"]
54
+ self.assertTrue(returned["returned"])
55
+ self.assertIsNone(returned["stop"])
56
+ self.assertEqual(returned["events"][-1]["kind"], "return")
57
+ self.assertFalse(stopped["returned"])
58
+ self.assertEqual(stopped["stop"], STOP)
59
+ # The stop names the addition that would have built the value, and every earlier addition keeps its event.
60
+ site = stopped["stopSite"]
61
+ self.assertGreater(site, c.labels["adds"])
62
+ self.assertEqual((site - c.labels["adds"]) % 2, 0)
63
+ sums = [e for e in stopped["events"] if e["kind"] == "arithmetic"]
64
+ self.assertEqual(len(sums), (site - c.labels["adds"]) // 2)
65
+ self.assertEqual(sums[-1]["site"], site - 2)
66
+ self.assertNotIn(site, [e["site"] for e in stopped["events"]])
67
+ # The steps were not spent, so the stop is not a step limit, and no path limit is recorded.
68
+ self.assertLess(stopped["steps"], 600 + 16)
69
+ self.assertEqual(r["gaps"], [])
70
+ self.assertFalse(r["completeWithinModel"])
71
+
72
+ def test_controls_reached_before_the_stop_are_kept(self):
73
+ c = additions(600)
74
+ tested = {"name": "tested", "kind": "reach", "at": {"site": c.labels["test"]}, "expect": "always"}
75
+ result = run(600, relationalControls=[tested])["relationalControls"]["controls"][0]
76
+ self.assertEqual(result["verdict"], "held")
77
+ self.assertEqual(result["occurrences"], 2)
78
+ self.assertEqual([p["stop"] for p in result["paths"]], [None, STOP])
79
+
80
+ def test_a_control_past_the_stop_stays_undecided(self):
81
+ c = additions(600)
82
+ done = {"name": "done", "kind": "reach", "at": {"site": c.labels["done"]}, "expect": "always"}
83
+ result = run(600, relationalControls=[done])["relationalControls"]["controls"][0]
84
+ self.assertEqual(result["verdict"], "undecided")
85
+ self.assertEqual(sorted(p["verdict"] for p in result["paths"]), ["held", "undecided"])
86
+
87
+ def test_a_query_error_inside_a_path_still_fails_the_run(self):
88
+ # A four-byte model on a near call without an encoded PUSH CS is an error in the query.
89
+ c = Code().branch("e8", "next").label("next").emit("cb")
90
+ model = [{"site": 0, "evidence": "synthetic model", "returnBytes": 4, "cases": [{}]}]
91
+ with self.assertRaisesRegex(ValueError, "Modeled return width differs"):
92
+ report(c, "trace", returnBytes=4, registers=FRAME, callModels=model)
93
+
94
+ def test_a_memory_probe_past_the_limit_is_unresolved(self):
95
+ # A probe observes the path, so a base too large to offset leaves its row unresolved.
96
+ # Two terms, then two per addition: a base of TERM_LIMIT terms, which the displacement's
97
+ # offset term takes past the limit.
98
+ term = ("neg", ("unknown", "x"))
99
+ for _ in range((TERM_LIMIT - 2) // 2):
100
+ term = ("add", term, ("unknown", "y"))
101
+ Value(16, term)
102
+
103
+ class Probed:
104
+ bits = 16
105
+
106
+ def segment(self, name):
107
+ return const(0x3000, 16)
108
+
109
+ def reg(self, name):
110
+ return Value(16, term)
111
+
112
+ def keys(self, segment, offset, width):
113
+ raise AssertionError("the offset cannot be formed")
114
+
115
+ address = {"segment": "ds", "base": "bx", "displacement": 2, "width": 2}
116
+ row = probe_memory(Probed(), "probe", address)
117
+ self.assertEqual(row["unresolved"], STOP)
118
+ self.assertIsNone(row["offset"])
119
+ self.assertNotIn("byteProducers", row)
120
+
121
+ def test_a_register_part_of_a_root_at_the_limit_is_an_unformed_row(self):
122
+ # Six terms for the loaded EAX and two per NOT: 509 of them leave EAX at the limit, and
123
+ # selecting AX, AL or AH from it adds the term that passes it.
124
+ c = negations(509)
125
+ r = negate(509)
126
+ path, = r["paths"]
127
+ self.assertTrue(path["returned"])
128
+ self.assertTrue(r["completeWithinModel"])
129
+ root = path["registers"]["eax"]
130
+ self.assertIsNotNone(root["expression"])
131
+ self.assertIn(c.labels["last"], root["producers"])
132
+ checkpoint, = (e for e in path["events"] if e["kind"] == "checkpoint")
133
+ self.assertEqual(path["events"][-1]["kind"], "return")
134
+ for snapshot in (path["registers"], checkpoint["registers"], path["events"][-1]["registers"]):
135
+ for name in ("ax", "al", "ah"):
136
+ self.assertEqual(snapshot[name], {"bits": 8 if name != "ax" else 16, "expression": None, "value": None,
137
+ "producers": root["producers"], "unresolved": STOP})
138
+ self.assertEqual(snapshot["bx"]["value"], 0x1234)
139
+
140
+ def test_a_path_that_stops_with_a_root_at_the_limit_keeps_its_report(self):
141
+ c = negations(510)
142
+ path, = negate(510)["paths"]
143
+ self.assertEqual(path["stop"], STOP)
144
+ self.assertEqual(path["stopSite"], c.labels["last"])
145
+ self.assertEqual(path["registers"]["ax"]["unresolved"], STOP)
146
+ self.assertIsNotNone(path["registers"]["eax"]["expression"])
147
+
148
+ def test_a_control_on_an_unformed_register_is_undecided(self):
149
+ def bound(count):
150
+ c = negations(count)
151
+ rule = {"name": "bound", "kind": "relation", "at": {"site": c.labels["done"], "event": "checkpoint"},
152
+ "op": "le", "left": {"field": "registers.ax"}, "right": 65535}
153
+ return negate(count, relationalControls=[rule])["relationalControls"]["controls"][0]
154
+
155
+ # The positive control: one NOT fewer leaves AX formed, and a word is at most 65535.
156
+ self.assertEqual(bound(508)["verdict"], "held")
157
+ result = bound(509)
158
+ self.assertEqual(result["verdict"], "undecided")
159
+ occurrence, = result["paths"][0]["occurrences"]
160
+ self.assertIn("was not formed: " + STOP, occurrence["reason"])
161
+
162
+ def test_an_entry_frame_whose_sp_cannot_be_formed_is_not_established(self):
163
+ # The entry SP's root starts at twelve terms, so 506 NOTs of ESP leave it at the limit; SP
164
+ # at the arrival is past it, and no offset is claimed.
165
+ c = Code().emit("66 f7 d4" * 506).label("entry").emit("c3")
166
+ data = c.bytes()
167
+ regions = [{"name": "synthetic", "start": 0, "end": len(data), "ip": 0, "segment": 0x1000, "resident": True,
168
+ "entries": [0, c.labels["entry"]], "evidence": "synthetic declared code extent"}]
169
+ r = report(c, "trace", regions=regions, entry=c.labels["entry"], registers={"ds": 0x3000, "ss": 0x9000},
170
+ entryFrame={"from": 0}, maxSteps=520)
171
+ frame = r["entryFrame"]
172
+ self.assertFalse(frame["established"])
173
+ self.assertEqual(frame["arrivals"], 1)
174
+ self.assertEqual(frame["reasons"], ["SP at an arrival could not be formed: " + STOP])
175
+
176
+ def test_an_entry_frame_whose_bp_cannot_be_formed_names_the_limit(self):
177
+ # BP copies the entry SP's twelve terms, so 506 NOTs of EBP leave it at the limit; BP at the
178
+ # arrival is past it. SP is unaffected, so the frame is still established.
179
+ def frame(count):
180
+ c = Code().emit("66 89 e5" + " 66 f7 d5" * count).label("entry").emit("c3")
181
+ data = c.bytes()
182
+ regions = [{"name": "synthetic", "start": 0, "end": len(data), "ip": 0, "segment": 0x1000, "resident": True,
183
+ "entries": [0, c.labels["entry"]], "evidence": "synthetic declared code extent"}]
184
+ return report(c, "trace", regions=regions, entry=c.labels["entry"], registers={"ds": 0x3000, "ss": 0x9000},
185
+ entryFrame={"from": 0}, maxSteps=520)["entryFrame"]
186
+
187
+ # The positive control: one NOT fewer forms BP, which is just not an offset.
188
+ formed = frame(505)
189
+ self.assertTrue(formed["established"])
190
+ self.assertIsNone(formed["bp"])
191
+ self.assertNotIn("bpUnresolved", formed)
192
+ unformed = frame(506)
193
+ self.assertTrue(unformed["established"])
194
+ self.assertIsNone(unformed["bp"])
195
+ self.assertEqual(unformed["reasons"], [])
196
+ self.assertEqual(unformed["bpUnresolved"], [STOP])
197
+
198
+ def test_a_dispatch_whose_index_cannot_be_formed_names_the_limit(self):
199
+ # EBX loads an unknown dword; 509 NOTs leave it at the limit, so BX at the jump is past it.
200
+ def outcome(count):
201
+ c = Code().emit("66 8b 1e 00 01" + " 66 f7 d3" * count).label("dispatch").emit("ff 27").label("table").emit("20 00")
202
+ data = c.bytes()
203
+ dispatch = {"site": c.labels["dispatch"], "inputRegister": "ax", "indexRegister": "bx", "inputs": [1],
204
+ "indexEvidence": "synthetic unknown index", "table": {"start": c.labels["table"], "count": 1,
205
+ "stride": 2, "width": 2, "countEvidence": "synthetic single entry", "offset": 0,
206
+ "mappingEvidence": "synthetic table"}}
207
+ return report(c, "dispatch", registers={"ds": 0x3000, "ss": 0x9000}, maxSteps=600,
208
+ dispatch=dispatch)["cases"][0]["outcomes"][0]
209
+
210
+ # The positive control: one NOT fewer forms BX, which is unknown, so the outcome only stops.
211
+ formed = outcome(508)
212
+ self.assertEqual(formed["status"], "unresolved")
213
+ self.assertNotIn("unresolved", formed)
214
+ unformed = outcome(509)
215
+ self.assertEqual(unformed["status"], "unresolved")
216
+ self.assertEqual(unformed["unresolved"], STOP)
217
+ # A NOT past the limit stops the path before the jump, which the stop already says.
218
+ stopped = outcome(510)
219
+ self.assertEqual(stopped["stop"], STOP)
220
+ self.assertNotIn("unresolved", stopped)
221
+
222
+ def test_the_error_survives_copy_and_pickle(self):
223
+ for rebuilt in (copy.copy(TermLimit()), pickle.loads(pickle.dumps(TermLimit()))):
224
+ self.assertIsInstance(rebuilt, TermLimit)
225
+ self.assertEqual(str(rebuilt), STOP)
226
+
227
+ def test_the_limit_outside_a_path_raises(self):
228
+ # Outside a traced path nothing can stop, so the limit fails the run as a ValueError.
229
+ value = const(1, 16)
230
+ with self.assertRaisesRegex(TermLimit, "term limit"):
231
+ for _ in range(TERM_LIMIT):
232
+ value = op("add", value, Value(16, ("unknown", "x")))
233
+ self.assertTrue(issubclass(TermLimit, ValueError))
234
+
235
+
236
+ if __name__ == "__main__":
237
+ unittest.main()
@@ -1,120 +0,0 @@
1
- """A value past the term limit stops its own path; the other paths and their controls stay."""
2
- import copy
3
- import pickle
4
- import unittest
5
- from test_x86 import Code, report
6
- from scientific_method_engine.x86.relational import probe_memory
7
- from scientific_method_engine.x86.values import TERM_LIMIT, TermLimit, Value, const, op
8
-
9
- FRAME = {"ds": 0x3000, "ss": 0x9000, "sp": 0xE000}
10
- STOP = f"expression term limit: a value's expression would hold more than {TERM_LIMIT} terms; narrow the query"
11
-
12
-
13
- def additions(count):
14
- """AX and BX load unknown words; a zero AX skips ``count`` additions of BX into AX."""
15
- c = Code().emit("a1 00 01 8b 1e 02 01").label("test").emit("85 c0 0f 84")
16
- c.fixups.append((len(c.data), 2, "done"))
17
- c.data.extend(bytes(2))
18
- c.label("adds")
19
- for _ in range(count):
20
- c.emit("01 d8")
21
- return c.label("done").emit("cb")
22
-
23
-
24
- def run(count, **extra):
25
- return report(additions(count), "trace", **{"returnBytes": 4, "registers": FRAME, "maxPaths": 2, "maxSteps": count + 16, **extra})
26
-
27
-
28
- class TermLimitTests(unittest.TestCase):
29
- def test_a_short_sum_returns_on_both_paths(self):
30
- r = run(8)
31
- self.assertEqual([p["returned"] for p in r["paths"]], [True, True])
32
- self.assertTrue(r["completeWithinModel"])
33
-
34
- def test_a_value_past_the_limit_stops_its_path_and_keeps_the_other(self):
35
- c = additions(600)
36
- r = run(600)
37
- returned, stopped = r["paths"]
38
- self.assertTrue(returned["returned"])
39
- self.assertIsNone(returned["stop"])
40
- self.assertEqual(returned["events"][-1]["kind"], "return")
41
- self.assertFalse(stopped["returned"])
42
- self.assertEqual(stopped["stop"], STOP)
43
- # The stop names the addition that would have built the value, and every earlier addition keeps its event.
44
- site = stopped["stopSite"]
45
- self.assertGreater(site, c.labels["adds"])
46
- self.assertEqual((site - c.labels["adds"]) % 2, 0)
47
- sums = [e for e in stopped["events"] if e["kind"] == "arithmetic"]
48
- self.assertEqual(len(sums), (site - c.labels["adds"]) // 2)
49
- self.assertEqual(sums[-1]["site"], site - 2)
50
- self.assertNotIn(site, [e["site"] for e in stopped["events"]])
51
- # The steps were not spent, so the stop is not a step limit, and no path limit is recorded.
52
- self.assertLess(stopped["steps"], 600 + 16)
53
- self.assertEqual(r["gaps"], [])
54
- self.assertFalse(r["completeWithinModel"])
55
-
56
- def test_controls_reached_before_the_stop_are_kept(self):
57
- c = additions(600)
58
- tested = {"name": "tested", "kind": "reach", "at": {"site": c.labels["test"]}, "expect": "always"}
59
- result = run(600, relationalControls=[tested])["relationalControls"]["controls"][0]
60
- self.assertEqual(result["verdict"], "held")
61
- self.assertEqual(result["occurrences"], 2)
62
- self.assertEqual([p["stop"] for p in result["paths"]], [None, STOP])
63
-
64
- def test_a_control_past_the_stop_stays_undecided(self):
65
- c = additions(600)
66
- done = {"name": "done", "kind": "reach", "at": {"site": c.labels["done"]}, "expect": "always"}
67
- result = run(600, relationalControls=[done])["relationalControls"]["controls"][0]
68
- self.assertEqual(result["verdict"], "undecided")
69
- self.assertEqual(sorted(p["verdict"] for p in result["paths"]), ["held", "undecided"])
70
-
71
- def test_a_query_error_inside_a_path_still_fails_the_run(self):
72
- # A four-byte model on a near call without an encoded PUSH CS is an error in the query.
73
- c = Code().branch("e8", "next").label("next").emit("cb")
74
- model = [{"site": 0, "evidence": "synthetic model", "returnBytes": 4, "cases": [{}]}]
75
- with self.assertRaisesRegex(ValueError, "Modeled return width differs"):
76
- report(c, "trace", returnBytes=4, registers=FRAME, callModels=model)
77
-
78
- def test_a_memory_probe_past_the_limit_is_unresolved(self):
79
- # A probe observes the path, so a base too large to offset leaves its row unresolved.
80
- # Two terms, then two per addition: a base of TERM_LIMIT terms, which the displacement's
81
- # offset term takes past the limit.
82
- term = ("neg", ("unknown", "x"))
83
- for _ in range((TERM_LIMIT - 2) // 2):
84
- term = ("add", term, ("unknown", "y"))
85
- Value(16, term)
86
-
87
- class Probed:
88
- bits = 16
89
-
90
- def segment(self, name):
91
- return const(0x3000, 16)
92
-
93
- def reg(self, name):
94
- return Value(16, term)
95
-
96
- def keys(self, segment, offset, width):
97
- raise AssertionError("the offset cannot be formed")
98
-
99
- address = {"segment": "ds", "base": "bx", "displacement": 2, "width": 2}
100
- row = probe_memory(Probed(), "probe", address)
101
- self.assertEqual(row["unresolved"], STOP)
102
- self.assertIsNone(row["offset"])
103
- self.assertNotIn("byteProducers", row)
104
-
105
- def test_the_error_survives_copy_and_pickle(self):
106
- for rebuilt in (copy.copy(TermLimit()), pickle.loads(pickle.dumps(TermLimit()))):
107
- self.assertIsInstance(rebuilt, TermLimit)
108
- self.assertEqual(str(rebuilt), STOP)
109
-
110
- def test_the_limit_outside_a_path_raises(self):
111
- # Outside a traced path nothing can stop, so the limit fails the run as a ValueError.
112
- value = const(1, 16)
113
- with self.assertRaisesRegex(TermLimit, "term limit"):
114
- for _ in range(TERM_LIMIT):
115
- value = op("add", value, Value(16, ("unknown", "x")))
116
- self.assertTrue(issubclass(TermLimit, ValueError))
117
-
118
-
119
- if __name__ == "__main__":
120
- unittest.main()