smda 2.0.2__tar.gz → 2.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. smda-2.2.0/PKG-INFO +140 -0
  2. {smda-2.0.2 → smda-2.2.0}/README.md +1 -0
  3. {smda-2.0.2 → smda-2.2.0}/setup.py +1 -1
  4. {smda-2.0.2 → smda-2.2.0}/smda/Disassembler.py +20 -7
  5. {smda-2.0.2 → smda-2.2.0}/smda/SmdaConfig.py +1 -1
  6. {smda-2.0.2 → smda-2.2.0}/smda/common/BinaryInfo.py +27 -4
  7. {smda-2.0.2 → smda-2.2.0}/smda/common/SmdaFunction.py +2 -2
  8. {smda-2.0.2 → smda-2.2.0}/smda/common/SmdaReport.py +11 -1
  9. {smda-2.0.2 → smda-2.2.0}/smda/common/labelprovider/ElfSymbolProvider.py +12 -8
  10. {smda-2.0.2 → smda-2.2.0}/smda/common/labelprovider/GoLabelProvider.py +7 -5
  11. {smda-2.0.2 → smda-2.2.0}/smda/common/labelprovider/PeSymbolProvider.py +26 -9
  12. {smda-2.0.2 → smda-2.2.0}/smda/ida/IdaInterface.py +119 -1
  13. smda-2.2.0/smda/utility/StringExtractor.py +158 -0
  14. smda-2.2.0/smda.egg-info/PKG-INFO +140 -0
  15. {smda-2.0.2 → smda-2.2.0}/smda.egg-info/SOURCES.txt +11 -1
  16. smda-2.2.0/tests/testBracketQueue.py +223 -0
  17. smda-2.2.0/tests/testEscaper.py +86 -0
  18. smda-2.2.0/tests/testFileFormatParsers.py +132 -0
  19. smda-2.2.0/tests/testIntegration.py +145 -0
  20. smda-2.2.0/tests/testTarjan.py +33 -0
  21. smda-2.0.2/PKG-INFO +0 -139
  22. smda-2.0.2/smda/utility/StringExtractor.py +0 -105
  23. smda-2.0.2/smda.egg-info/PKG-INFO +0 -139
  24. {smda-2.0.2 → smda-2.2.0}/LICENSE +0 -0
  25. {smda-2.0.2 → smda-2.2.0}/setup.cfg +0 -0
  26. {smda-2.0.2 → smda-2.2.0}/smda/DisassemblyResult.py +0 -0
  27. {smda-2.0.2 → smda-2.2.0}/smda/DisassemblyStatistics.py +0 -0
  28. {smda-2.0.2 → smda-2.2.0}/smda/__init__.py +0 -0
  29. {smda-2.0.2 → smda-2.2.0}/smda/cil/CilDisassembler.py +0 -0
  30. {smda-2.0.2 → smda-2.2.0}/smda/cil/CilInstructionEscaper.py +0 -0
  31. {smda-2.0.2 → smda-2.2.0}/smda/cil/FunctionAnalysisState.py +0 -0
  32. {smda-2.0.2 → smda-2.2.0}/smda/cil/__init__.py +0 -0
  33. {smda-2.0.2 → smda-2.2.0}/smda/common/BasicBlock.py +0 -0
  34. {smda-2.0.2 → smda-2.2.0}/smda/common/BlockLocator.py +0 -0
  35. {smda-2.0.2 → smda-2.2.0}/smda/common/CodeXref.py +0 -0
  36. {smda-2.0.2 → smda-2.2.0}/smda/common/DominatorTree.py +0 -0
  37. {smda-2.0.2 → smda-2.2.0}/smda/common/SmdaBasicBlock.py +0 -0
  38. {smda-2.0.2 → smda-2.2.0}/smda/common/SmdaInstruction.py +0 -0
  39. {smda-2.0.2 → smda-2.2.0}/smda/common/TailcallAnalyzer.py +0 -0
  40. {smda-2.0.2 → smda-2.2.0}/smda/common/Tarjan.py +0 -0
  41. {smda-2.0.2 → smda-2.2.0}/smda/common/__init__.py +0 -0
  42. {smda-2.0.2 → smda-2.2.0}/smda/common/labelprovider/AbstractLabelProvider.py +0 -0
  43. {smda-2.0.2 → smda-2.2.0}/smda/common/labelprovider/CilSymbolProvider.py +0 -0
  44. {smda-2.0.2 → smda-2.2.0}/smda/common/labelprovider/DelphiKbSymbolProvider.py +0 -0
  45. {smda-2.0.2 → smda-2.2.0}/smda/common/labelprovider/ElfApiResolver.py +0 -0
  46. {smda-2.0.2 → smda-2.2.0}/smda/common/labelprovider/OrdinalHelper.py +0 -0
  47. {smda-2.0.2 → smda-2.2.0}/smda/common/labelprovider/PdbSymbolProvider.py +0 -0
  48. {smda-2.0.2 → smda-2.2.0}/smda/common/labelprovider/WinApiResolver.py +0 -0
  49. {smda-2.0.2 → smda-2.2.0}/smda/common/labelprovider/__init__.py +0 -0
  50. {smda-2.0.2 → smda-2.2.0}/smda/ida/BackendInterface.py +0 -0
  51. {smda-2.0.2 → smda-2.2.0}/smda/ida/IdaExporter.py +0 -0
  52. {smda-2.0.2 → smda-2.2.0}/smda/ida/__init__.py +0 -0
  53. {smda-2.0.2 → smda-2.2.0}/smda/intel/BitnessAnalyzer.py +0 -0
  54. {smda-2.0.2 → smda-2.2.0}/smda/intel/FunctionAnalysisState.py +0 -0
  55. {smda-2.0.2 → smda-2.2.0}/smda/intel/FunctionCandidate.py +0 -0
  56. {smda-2.0.2 → smda-2.2.0}/smda/intel/FunctionCandidateManager.py +0 -0
  57. {smda-2.0.2 → smda-2.2.0}/smda/intel/IndirectCallAnalyzer.py +0 -0
  58. {smda-2.0.2 → smda-2.2.0}/smda/intel/IntelDisassembler.py +0 -0
  59. {smda-2.0.2 → smda-2.2.0}/smda/intel/IntelInstructionEscaper.py +0 -0
  60. {smda-2.0.2 → smda-2.2.0}/smda/intel/JumpTableAnalyzer.py +0 -0
  61. {smda-2.0.2 → smda-2.2.0}/smda/intel/LanguageAnalyzer.py +0 -0
  62. {smda-2.0.2 → smda-2.2.0}/smda/intel/MnemonicTfIdf.py +0 -0
  63. {smda-2.0.2 → smda-2.2.0}/smda/intel/__init__.py +0 -0
  64. {smda-2.0.2 → smda-2.2.0}/smda/intel/definitions.py +0 -0
  65. {smda-2.0.2 → smda-2.2.0}/smda/utility/BracketQueue.py +0 -0
  66. {smda-2.0.2 → smda-2.2.0}/smda/utility/DelphiKbFileLoader.py +0 -0
  67. {smda-2.0.2 → smda-2.2.0}/smda/utility/ElfFileLoader.py +0 -0
  68. {smda-2.0.2 → smda-2.2.0}/smda/utility/FileLoader.py +0 -0
  69. {smda-2.0.2 → smda-2.2.0}/smda/utility/MachoFileLoader.py +0 -0
  70. {smda-2.0.2 → smda-2.2.0}/smda/utility/MemoryFileLoader.py +0 -0
  71. {smda-2.0.2 → smda-2.2.0}/smda/utility/PeFileLoader.py +0 -0
  72. {smda-2.0.2 → smda-2.2.0}/smda/utility/PriorityQueue.py +0 -0
  73. {smda-2.0.2 → smda-2.2.0}/smda/utility/__init__.py +0 -0
  74. {smda-2.0.2 → smda-2.2.0}/smda.egg-info/dependency_links.txt +0 -0
  75. {smda-2.0.2 → smda-2.2.0}/smda.egg-info/requires.txt +0 -0
  76. {smda-2.0.2 → smda-2.2.0}/smda.egg-info/top_level.txt +0 -0
smda-2.2.0/PKG-INFO ADDED
@@ -0,0 +1,140 @@
1
+ Metadata-Version: 2.1
2
+ Name: smda
3
+ Version: 2.2.0
4
+ Summary: A recursive disassmbler optimized for CFG recovery from memory dumps. Based on capstone.
5
+ Home-page: https://github.com/danielplohmann/smda
6
+ Author: Daniel Plohmann
7
+ Author-email: daniel.plohmann@mailbox.org
8
+ License: BSD 2-Clause
9
+ Classifier: Development Status :: 4 - Beta
10
+ Classifier: License :: OSI Approved :: BSD License
11
+ Classifier: Operating System :: OS Independent
12
+ Classifier: Programming Language :: Python :: 2.7
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Topic :: Security
15
+ Classifier: Topic :: Software Development :: Disassemblers
16
+ Description-Content-Type: text/markdown
17
+ License-File: LICENSE
18
+
19
+
20
+ # SMDA
21
+
22
+ SMDA is a minimalist recursive disassembler library that is optimized for accurate Control Flow Graph (CFG) recovery from memory dumps.
23
+ It is based on [Capstone](http://www.capstone-engine.org/) and currently supports x86/x64 Intel machine code.
24
+ As input, arbitrary memory dumps (ideally with known base address) can be processed.
25
+ The output is a collection of functions, basic blocks, and instructions with their respective edges between blocks and functions (in/out).
26
+ Optionally, references to the Windows API can be inferred by using the ApiScout method.
27
+
28
+ ## Installation
29
+
30
+ With version 1.2.0, we have finally simplified things by moving to [PyPI](https://pypi.org/project/smda/)!
31
+ So installation now is as easy as:
32
+
33
+ ```
34
+ $ pip install smda
35
+ ```
36
+
37
+ ## Usage
38
+
39
+ A typical workflow using SMDA could like this:
40
+
41
+ ```
42
+ >>> from smda.Disassembler import Disassembler
43
+ >>> disassembler = Disassembler()
44
+ >>> report = disassembler.disassembleFile("/bin/cat")
45
+ >>> print(report)
46
+ 0.777s -> (architecture: intel.64bit, base_addr: 0x00000000): 143 functions
47
+ >>> for fn in report.getFunctions():
48
+ ... print(fn)
49
+ ... for ins in fn.getInstructions():
50
+ ... print(ins)
51
+ ...
52
+ 0x00001720: (-> 1, 1->) 3 blocks, 7 instructions.
53
+ 0x00001720: ( 4883ec08) - sub rsp, 8
54
+ 0x00001724: (488b05bd682000) - mov rax, qword ptr [rip + 0x2068bd]
55
+ 0x0000172b: ( 4885c0) - test rax, rax
56
+ 0x0000172e: ( 7402) - je 0x1732
57
+ 0x00001730: ( ffd0) - call rax
58
+ 0x00001732: ( 4883c408) - add rsp, 8
59
+ 0x00001736: ( c3) - ret
60
+ 0x00001ad0: (-> 1, 4->) 1 blocks, 12 instructions.
61
+ [...]
62
+ >>> json_report = report.toDict()
63
+ ```
64
+
65
+ There is also a demo script:
66
+
67
+ * analyze.py -- example usage: perform disassembly on a file or memory dump and optionally store results in JSON to a given output path.
68
+
69
+ The code should be fully compatible with Python 3.8+.
70
+ Further explanation on the innerworkings follow in separate publications but will be referenced here.
71
+
72
+ To take full advantage of SMDA's capabilities, make sure to (optionally) install:
73
+ * lief
74
+ * pdbparse (currently as fork from https://github.com/VPaulV/pdbparse to support Python3)
75
+
76
+ ## Version History
77
+ * 2025-06-13: v2.1.0 - Support for export from IDA 9.0+ (THX to @jershmagersh for the update!).
78
+ * 2025-02-26: v2.0.2 - Adjusting relative import, adding init file.
79
+ * 2025-02-25: v2.0.0 - Initial experimental support for CIL (.NET) disassembly.
80
+ * 2025-02-24: v1.14.3 - PicHashing can now be disabled via SmdaConfig to save some processing time. (THX to @Nalexander-hanel!)
81
+ * 2025-02-24: v1.14.2 - We are Python 3.8+ compatible again (changed UTC usage) and (DWARF) PE symbols for PE files should be extracted again (THX to @N0fix for the update!)
82
+ * 2025-02-21: v1.14.1 - Fixed changed field names in LIEF usage that broke ELF parsing, added tests for ELF+macOS parsing (THX to @N0fix for the update!)
83
+ * 2025-01-29: v1.14.0 - Bump to LIEF 0.16.0+ (THX to @huettenhain for the ping!). Migrated tests to `pytest`, UTC datetime handling fixes.
84
+ * 2025-01-26: v1.13.24 - Added functionality to import and export SMDA reports as JSON. Fixed byte patterns matching special regex chars (THX to @alexander-hanel!).
85
+ * 2024-07-26: v1.13.23 - Now using OEP as symbol function candidate when available (THX to @alexander-hanel for reporting!).
86
+ * 2024-05-10: v1.13.22 - Handled odd case where disassembly with capstone and IDA would return different results (THX to @r0ny123 for reporting!).
87
+ * 2024-04-17: v1.13.21 - Fixed handling of Go binaries for version 1.20+ (THX to @Manny684!).
88
+ * 2024-04-08: v1.13.20 - Fixed handling of bnd prefix in CFG instructions to help with parsing PLT (THX to @Manny684!).
89
+ * 2024-04-02: v1.13.19 - Fixed bug in string parsing, added tests, strings now no longer are hex-encoded as they are always printable anyway.
90
+ * 2024-03-12: v1.13.18 - Added functionality to extract and store all referenced strings along SmdaFunctions (has to be enabled via SmdaConfig).
91
+ * 2024-03-12: v1.13.17 - Extended disassembleBuffer() to now take additional arguments `code_areas` and `oep`.
92
+ * 2024-02-21: v1.13.16 - BREAKING IntelInstructionEscaper.escapeMnemonic: Escaper now handles another 200 instruction names found in other capstone source files (THX for reporting @malwarefrank!).
93
+ * 2024-02-15: v1.13.15 - Fixed issues with version recognition in SmdaFunction which cause issues in MCRIT (THX to @
94
+ malwarefrank!)
95
+ * 2024-02-02: v1.13.12 - Versions might be non-numerical, addressed that in SmdaFunction.
96
+ * 2024-01-23: v1.13.11 - Introduced indicator in SmdaConfig for compatibility of instruction escaping.
97
+ * 2024-01-23: v1.13.10 - Parsing of PE files should work again with lief >=0.14.0.
98
+ * 2024-01-23: v1.13.9 - Improved parsing robustness for section/segment tables in ELF files, also now padding with zeroes when finding less content than expected physical size in a segment (THX for reporting @schrodyn!).
99
+ * 2024-01-23: v1.13.8 - BREAKING adjustments to IntelInstructionEscaper.escapeMnemonic: Escaper now is capable of handling all known x86/x64 instructions in capstone (THX for reporting @schrodyn!).
100
+ * 2023-12-01: v1.13.7 - Skip processing of Delphi structs for large files, workaround until this is properly reimplemented.
101
+ * 2023-11-29: v1.13.6 - Made OpcodeHash an attribute with on-demand calculation to save processing time.
102
+ * 2023-11-29: v1.13.3 - Implemented an alternative queue working with reference count based brackets in pursuit of accelerated processing.
103
+ * 2023-11-28: v1.13.2 - IndirectCallAnalyzer will now analyze at most a configurable amount of calls per basic block, default 50.
104
+ * 2023-11-21: v1.13.1 - SmdaBasicBlock now has `getPredecessors()` and `getSuccessors()`.
105
+ * 2023-11-21: v1.13.0 - BREAKING adjustments to PicHashing (now wildcarding intraprocedural jumps in functions, additionally more immediates if within address space). Introduction of OpcodeHash (OpcHash), which wildcards all but prefixes and opcode bytes.
106
+ * 2023-10-12: v1.12.7 - Bugfix for parsing Delphi structs.
107
+ * 2023-09-15: v1.12.6 - Bugfix in BlockLocator (THX to @cccs-ay!).
108
+ * 2023-08-28: v1.12.5 - Bugfix for address dereferencing where buffer sizes were not properly checked (THX to @yankovs!).
109
+ * 2023-08-08: v1.12.4 - SmdaBasicBlock can now do getPicBlockHash().
110
+ * 2023-05-23: v1.12.3 - Fixed bugs in PE parser and Go parser.
111
+ * 2023-05-08: v1.12.1 - Get rid of deprecation warning in IDA 8.0+.
112
+ * 2023-03-24: v1.12.0 - SMDA now parses PE export directories for symbols, as well as MinGW DWARF information if available.
113
+ * 2023-03-14: v1.11.2 - SMDA report now also contains SHA1 and MD5.
114
+ * 2023-03-14: v1.11.1 - rendering dotGraph can now include API references instead of plain calls.
115
+ * 2023-02-06: v1.11.0 - SmdaReport now has functionality to find a function/block by a given offset contained within in (THX to @cccs-ay!).
116
+ * 2023-02-06: v1.10.0 - Adjusted to LIEF 0.12.3 API for binary parsing (THX to @lainswork!).
117
+ * 2022-08-12: v1.9.1 - Added support for parsing intel MachO files, including Go parsing.
118
+ * 2022-08-01: v1.8.0 - Added support for parsing Go function information (THX to @danielenders1!).
119
+ * 2022-01-27: v1.7.0 - SmdaReports now contains a field `oep`; SmdaFunctions now indicate `is_exported` and can provide CodeXrefs via `getCodeInrefs()` and `getCodeOutrefs()`. (THX for the ideas: @mr-tz)
120
+ * 2021-08-20: v1.6.0 - Bugfix for alignment calculation of binary mappings. (THX: @williballenthin)
121
+ * 2021-08-19: v1.6.0 - Bugfix for truncation during ELF segment/section loading. API usage in ELF files is now resolved as well! (THX: @williballenthin)
122
+ * 2020-10-30: v1.5.0 - PE section table now contained in SmdaReport and added `SmdaReport.getSection(offset)`.
123
+ * 2020-10-26: v1.4.0 - Adding SmdaBasicBlock. Some convenience code to ease intgration with capa. (GeekWeek edition!)
124
+ * 2020-06-22: v1.3.0 - Added DominatorTree (Implementation by Armin Rigo) to calculate function nesting depth, shortened PIC hash to 8 byte, added some missing instructions for the InstructionEscaper, IdaInterface now demangles names.
125
+ * 2020-04-29: v1.2.0 - Restructured config.py into smda/SmdaConfig.py to similfy usage and now available via PyPI! The smda/Disassembler.py now emits a report object (smda.common.SmdaReport) that allows direct (pythonic) interaction with the results - a JSON can still be easily generated by using toDict() on the report.
126
+ * 2020-04-28: v1.1.0 - Several improvements, including: x64 jump table handling, better data flow handling for calls using registers and tailcalls, extended list of common prologues based on much more groundtruth data, extended padding instruction list for gap function discovery, adjusted weights in candidate priority score, filtering code areas based on section tables, using exported symbols as candidates, new function output metadata: confidence score based on instruction mnemonic histogram, PIC hash based on escaped binary instruction sequence
127
+ * 2018-07-01: v1.0.0 - Initial Release.
128
+
129
+
130
+ ## Credits
131
+
132
+ Thanks to Steffen Enders for his extensive contributions to this project!
133
+ Thanks to Paul Hordiienko for adding symbol parsing support (ELF+PDB)!
134
+ Thanks to Jonathan Crussell for helping me to beef up SMDA enough to make it a disassembler backend in capa!
135
+ Thanks to Willi Ballenthin for improving handling of ELF files, including properly handling API usage!
136
+ Thanks to Daniel Enders for his contributions to the parsing of the Golang function registry and label information!
137
+ The project uses the implementation of Tarjan's Algorithm by Bas Westerbaan and the implementation of Lengauer-Tarjan's Algorithm for the DominatorTree by Armin Rigo.
138
+
139
+ Pull requests welcome! :)
140
+
@@ -56,6 +56,7 @@ To take full advantage of SMDA's capabilities, make sure to (optionally) install
56
56
  * pdbparse (currently as fork from https://github.com/VPaulV/pdbparse to support Python3)
57
57
 
58
58
  ## Version History
59
+ * 2025-06-13: v2.1.0 - Support for export from IDA 9.0+ (THX to @jershmagersh for the update!).
59
60
  * 2025-02-26: v2.0.2 - Adjusting relative import, adding init file.
60
61
  * 2025-02-25: v2.0.0 - Initial experimental support for CIL (.NET) disassembly.
61
62
  * 2025-02-24: v1.14.3 - PicHashing can now be disabled via SmdaConfig to save some processing time. (THX to @Nalexander-hanel!)
@@ -19,7 +19,7 @@ else:
19
19
  setup(
20
20
  name='smda',
21
21
  # note to self: always change this in config as well.
22
- version='2.0.2',
22
+ version='2.2.0',
23
23
  description='A recursive disassmbler optimized for CFG recovery from memory dumps. Based on capstone.',
24
24
  long_description_content_type="text/markdown",
25
25
  long_description=long_description,
@@ -10,6 +10,7 @@ from smda.utility.MemoryFileLoader import MemoryFileLoader
10
10
  from smda.utility.StringExtractor import extract_strings
11
11
  from smda.SmdaConfig import SmdaConfig
12
12
  from smda.common.BinaryInfo import BinaryInfo
13
+ from smda.common.labelprovider.GoLabelProvider import GoSymbolProvider
13
14
  from smda.common.SmdaReport import SmdaReport
14
15
  from smda.intel.IntelDisassembler import IntelDisassembler
15
16
  from smda.cil.CilDisassembler import CilDisassembler
@@ -54,12 +55,18 @@ class Disassembler(object):
54
55
  LOGGER.debug("Current analysis callback time %s", (time_diff))
55
56
  return time_diff.seconds >= self._timeout
56
57
 
57
- def _addStringsToReport(self, smda_report, buffer):
58
+ def _addStringsToReport(self, smda_report, buffer, mode=None):
58
59
  smda_report.buffer = buffer
59
60
  for smda_function in smda_report.getFunctions():
60
- function_strings = {}
61
- for string, addr in extract_strings(smda_function):
62
- function_strings[addr] = string
61
+ function_strings = []
62
+ for string_result in extract_strings(smda_function, mode=mode):
63
+ string, referencing_addr, string_addr, string_type = string_result
64
+ function_strings.append({
65
+ "string": string,
66
+ "ins_addr": referencing_addr,
67
+ "data_addr": string_addr,
68
+ "type": string_type
69
+ })
63
70
  smda_function.stringrefs = function_strings
64
71
 
65
72
  def disassembleFile(self, file_path, pdb_path=""):
@@ -82,7 +89,9 @@ class Disassembler(object):
82
89
  self.disassembler.addPdbFile(binary_info, pdb_path)
83
90
  smda_report = self._disassemble(binary_info, timeout=self.config.TIMEOUT)
84
91
  if self.config.WITH_STRINGS:
85
- self._addStringsToReport(smda_report, file_content)
92
+ is_go_binary = GoSymbolProvider(None).getPcLntabOffset(binary_info.binary)
93
+ string_mode = "go" if is_go_binary else None
94
+ self._addStringsToReport(smda_report, file_content, mode=string_mode)
86
95
  if self.config.STORE_BUFFER:
87
96
  smda_report.buffer = file_content
88
97
  except Exception as exc:
@@ -110,7 +119,9 @@ class Disassembler(object):
110
119
  try:
111
120
  smda_report = self._disassemble(binary_info, timeout=self.config.TIMEOUT)
112
121
  if self.config.WITH_STRINGS:
113
- self._addStringsToReport(smda_report, file_content)
122
+ is_go_binary = GoSymbolProvider(None).getPcLntabOffset(binary_info.binary)
123
+ string_mode = "go" if is_go_binary else None
124
+ self._addStringsToReport(smda_report, file_content, mode=string_mode)
114
125
  if self.config.STORE_BUFFER:
115
126
  smda_report.buffer = file_content
116
127
  except Exception as exc:
@@ -136,7 +147,9 @@ class Disassembler(object):
136
147
  try:
137
148
  smda_report = self._disassemble(binary_info, timeout=self.config.TIMEOUT)
138
149
  if self.config.WITH_STRINGS:
139
- self._addStringsToReport(smda_report, file_content)
150
+ is_go_binary = GoSymbolProvider(None).getPcLntabOffset(binary_info.binary)
151
+ string_mode = "go" if is_go_binary else None
152
+ self._addStringsToReport(smda_report, file_content, mode=string_mode)
140
153
  if self.config.STORE_BUFFER:
141
154
  smda_report.buffer = file_content
142
155
  except Exception as exc:
@@ -5,7 +5,7 @@ import logging
5
5
  class SmdaConfig(object):
6
6
 
7
7
  # note to self: always change this in setup.py as well!
8
- VERSION = "2.0.2"
8
+ VERSION = "2.2.0"
9
9
  ESCAPER_DOWNWARD_COMPATIBILITY = "1.13.16"
10
10
  CONFIG_FILE_PATH = str(os.path.abspath(__file__))
11
11
  PROJECT_ROOT = str(os.path.abspath(os.sep.join([CONFIG_FILE_PATH, "..", ".."])))
@@ -3,6 +3,9 @@ import hashlib
3
3
  import lief
4
4
  lief.logging.disable()
5
5
 
6
+ from smda.common.labelprovider.PeSymbolProvider import PeSymbolProvider
7
+ from smda.common.labelprovider.ElfSymbolProvider import ElfSymbolProvider
8
+
6
9
 
7
10
  class BinaryInfo(object):
8
11
  """ simple DTO to contain most information related to the binary/buffer to be analyzed """
@@ -24,6 +27,7 @@ class BinaryInfo(object):
24
27
  md5 = ""
25
28
  version = ""
26
29
  exported_functions = None
30
+ imported_functions = None
27
31
  oep = None
28
32
 
29
33
  def __init__(self, binary):
@@ -46,11 +50,21 @@ class BinaryInfo(object):
46
50
  def getExportedFunctions(self):
47
51
  if self.exported_functions is None:
48
52
  lief_result = lief.parse(self.raw_data)
49
- if isinstance(lief_result, lief.PE.Binary) or isinstance(lief_result, lief.ELF.Binary):
50
- self.exported_functions = {}
51
- for function in lief_result.exported_functions:
52
- self.exported_functions[function.address] = function.name
53
+ if isinstance(lief_result, lief.PE.Binary):
54
+ self.exported_functions = PeSymbolProvider(None).parseExports(lief_result)
55
+ elif isinstance(lief_result, lief.ELF.Binary):
56
+ self.exported_functions = ElfSymbolProvider(None).parseExports(lief_result)
53
57
  return self.exported_functions
58
+
59
+ def getImportedFunctions(self):
60
+ if self.imported_functions is None:
61
+ lief_result = lief.parse(self.raw_data)
62
+ if isinstance(lief_result, lief.PE.Binary):
63
+ imports = PeSymbolProvider(None).parseSymbols(lief_result)
64
+ self.imported_functions = PeSymbolProvider(None).parseImports(lief_result)
65
+ elif isinstance(lief_result, lief.ELF.Binary):
66
+ self.imported_functions = ElfSymbolProvider(None).parseSymbols(lief_result.dynamic_symbols)
67
+ return self.imported_functions
54
68
 
55
69
  def getSections(self):
56
70
  pefile = lief.parse(self.raw_data)
@@ -75,3 +89,12 @@ class BinaryInfo(object):
75
89
  else:
76
90
  is_inside = any([a[0] <= address < a[1] for a in self.code_areas])
77
91
  return is_inside
92
+
93
+ def getHeaderBytes(self):
94
+ if self.raw_data:
95
+ lief_result = lief.parse(self.raw_data)
96
+ if isinstance(lief_result, lief.PE.Binary):
97
+ return self.raw_data[:0x400]
98
+ elif isinstance(lief_result, lief.ELF.Binary):
99
+ return self.raw_data[:0x40]
100
+ return None
@@ -227,7 +227,7 @@ class SmdaFunction(object):
227
227
  smda_function.pic_hash = function_dict["metadata"]["pic_hash"] if "pic_hash" in function_dict["metadata"] else None
228
228
  smda_function.strongly_connected_components = function_dict["metadata"]["strongly_connected_components"]
229
229
  smda_function.tfidf = function_dict["metadata"]["tfidf"]
230
- smda_function.stringrefs = {addr: string for addr, string in function_dict["stringrefs"].items()} if "stringrefs" in function_dict else {}
230
+ smda_function.stringrefs = function_dict["stringrefs"] if "stringrefs" in function_dict else {}
231
231
  if binary_info and binary_info.architecture:
232
232
  smda_function._escaper = IntelInstructionEscaper if binary_info.architecture in ["intel"] else None
233
233
  else:
@@ -264,7 +264,7 @@ class SmdaFunction(object):
264
264
  "offset": self.offset,
265
265
  "blocks": blocks_as_dict,
266
266
  "apirefs": self.apirefs,
267
- "stringrefs": {addr: string for addr, string in self.stringrefs.items()} if self.stringrefs is not None else {},
267
+ "stringrefs": self.stringrefs if self.stringrefs is not None else {},
268
268
  "blockrefs": self.blockrefs,
269
269
  "inrefs": self.inrefs,
270
270
  "outrefs": self.outrefs,
@@ -55,6 +55,7 @@ class SmdaReport(object):
55
55
  timestamp = None
56
56
  version = None
57
57
  xcfg = None
58
+ xheader = None
58
59
 
59
60
  # on first usage, initialize codexrefs objects for all functions based on inrefs/outrefs (requires knowledge about all functions)
60
61
  _has_codexrefs = False
@@ -95,6 +96,11 @@ class SmdaReport(object):
95
96
  self.timestamp = datetime.datetime.now(datetime.timezone.utc)
96
97
  self.version = disassembly.binary_info.version
97
98
  self.xcfg = self._convertCfg(disassembly, config=config)
99
+ self.xheader = disassembly.binary_info.getHeaderBytes()
100
+ self.xmetadata = {
101
+ "exported_functions": disassembly.binary_info.getExportedFunctions(),
102
+ "imported_functions": disassembly.binary_info.getImportedFunctions(),
103
+ }
98
104
 
99
105
  def _convertCfg(self, disassembly, config=None):
100
106
  function_results = {}
@@ -257,6 +263,8 @@ class SmdaReport(object):
257
263
  binary_info.binary_size = smda_report.binary_size
258
264
  binary_info.oep = smda_report.oep
259
265
  smda_report.xcfg = {int(function_addr): SmdaFunction.fromDict(function_dict, binary_info=binary_info, version=smda_report.smda_version, smda_report=smda_report) for function_addr, function_dict in report_dict["xcfg"].items()}
266
+ smda_report.xheader = bytes.fromhex(report_dict["xheader"]) if "xheader" in report_dict else None
267
+ smda_report.xmetadata = report_dict["xmetadata"] if "xmetadata" in report_dict else None
260
268
  return smda_report
261
269
 
262
270
  def toDict(self) -> dict:
@@ -289,7 +297,9 @@ class SmdaReport(object):
289
297
  "statistics": self.statistics.toDict(),
290
298
  "status": self.status,
291
299
  "timestamp": self.timestamp.strftime("%Y-%m-%dT%H-%M-%S"),
292
- "xcfg": {function_addr: smda_function.toDict() for function_addr, smda_function in self.xcfg.items()}
300
+ "xcfg": {function_addr: smda_function.toDict() for function_addr, smda_function in self.xcfg.items()},
301
+ "xheader": self.xheader.hex() if self.xheader else "",
302
+ "xmetadata": self.xmetadata,
293
303
  }
294
304
 
295
305
  @classmethod
@@ -46,27 +46,31 @@ class ElfSymbolProvider(AbstractLabelProvider):
46
46
  lief_binary = lief.parse(data)
47
47
  self._parseOep(lief_binary)
48
48
  # TODO split resolution into API/dynamic part and local symbols
49
- self._parseExports(lief_binary)
50
- self._parseSymbols(lief_binary.symtab_symbols)
51
- self._parseSymbols(lief_binary.dynamic_symbols)
49
+ self._func_symbols.update(self.parseExports(lief_binary))
50
+ self._func_symbols.update(self.parseSymbols(lief_binary.symtab_symbols))
51
+ self._func_symbols.update(self.parseSymbols(lief_binary.dynamic_symbols))
52
52
  for reloc in lief_binary.relocations:
53
53
  if reloc.has_symbol:
54
54
  self._func_symbols[reloc.address] = reloc.symbol.name
55
55
 
56
- def _parseExports(self, binary):
56
+ def parseExports(self, binary):
57
+ function_symbols = {}
57
58
  for function in binary.exported_functions:
58
- self._func_symbols[function.address] = function.name
59
+ function_symbols[function.address] = function.name
60
+ return function_symbols
59
61
 
60
- def _parseSymbols(self, symbols):
62
+ def parseSymbols(self, symbols):
63
+ function_symbols = {}
61
64
  for symbol in symbols:
62
- if symbol.is_function:
65
+ if symbol is not None and symbol.is_function:
63
66
  if symbol.value != 0:
64
67
  func_name = ""
65
68
  try:
66
69
  func_name = symbol.demangled_name
67
70
  except:
68
71
  func_name = symbol.name
69
- self._func_symbols[symbol.value] = func_name
72
+ function_symbols[symbol.value] = func_name
73
+ return function_symbols
70
74
 
71
75
  def getSymbol(self, address):
72
76
  return self._func_symbols.get(address, "")
@@ -19,8 +19,7 @@ class GoSymbolProvider(AbstractLabelProvider):
19
19
  # addr:func_name
20
20
  self._func_symbols = {}
21
21
 
22
- def update(self, binary_info):
23
- binary = binary_info.binary
22
+ def getPcLntabOffset(self, binary):
24
23
  pclntab_offset = None
25
24
  try:
26
25
  lief_binary = lief.parse(binary)
@@ -37,10 +36,13 @@ class GoSymbolProvider(AbstractLabelProvider):
37
36
  # scan for offset of structure
38
37
  pclntab_regex = re.compile(b".\xFF\xFF\xFF\x00\x00\x01(\x04|\x08)")
39
38
  hits = [match.start() for match in re.finditer(pclntab_regex, binary)]
40
- if len(hits) > 1:
41
- logging.error("GoLabelProvider found too many candidates for pclntab")
42
- elif len(hits) == 1:
39
+ if len(hits) == 1:
43
40
  pclntab_offset = hits[0]
41
+ return pclntab_offset is not None
42
+
43
+ def update(self, binary_info):
44
+ binary = binary_info.binary
45
+ pclntab_offset = self.getPcLntabOffset(binary)
44
46
  # if we found a valid offset, do the pclntab parsing
45
47
  if pclntab_offset:
46
48
  try:
@@ -2,6 +2,7 @@
2
2
 
3
3
  import logging
4
4
  from .AbstractLabelProvider import AbstractLabelProvider
5
+ from smda.common.labelprovider.OrdinalHelper import OrdinalHelper
5
6
 
6
7
  LOGGER = logging.getLogger(__name__)
7
8
 
@@ -42,11 +43,12 @@ class PeSymbolProvider(AbstractLabelProvider):
42
43
  lief_binary = lief.parse(binary_info.file_path)
43
44
  if lief_binary is not None:
44
45
  self._parseOep(lief_binary)
45
- self._parseExports(lief_binary)
46
- self._parseSymbols(lief_binary)
46
+ self._func_symbols.update(self.parseExports(lief_binary))
47
+ self._func_symbols.update(self.parseSymbols(lief_binary))
47
48
 
48
- def _parseExports(self, binary):
49
- for function in binary.exported_functions:
49
+ def parseExports(self, lief_binary):
50
+ function_symbols = {}
51
+ for function in lief_binary.exported_functions:
50
52
  function_name = ""
51
53
  try:
52
54
  # here may occur a LIEF exception that we want to skip ->
@@ -54,11 +56,13 @@ class PeSymbolProvider(AbstractLabelProvider):
54
56
  function_name = function.name
55
57
  except:
56
58
  pass
57
- if function_name and all(c in range(0x20, 0x7f) for c in function_name):
58
- self._func_symbols[binary.imagebase + function.address] = function_name
59
+ if function_name and all(ord(c) in range(0x20, 0x7f) for c in function_name):
60
+ function_symbols[lief_binary.imagebase + function.address] = function_name
61
+ return function_symbols
59
62
 
60
- def _parseSymbols(self, lief_binary):
63
+ def parseSymbols(self, lief_binary):
61
64
  # find VA of first code section
65
+ function_symbols = {}
62
66
  code_base_address = None
63
67
  for section in lief_binary.sections:
64
68
  if section.characteristics & 0x20000000:
@@ -78,8 +82,21 @@ class PeSymbolProvider(AbstractLabelProvider):
78
82
  if function_name and all(ord(c) in range(0x20, 0x7f) for c in function_name):
79
83
  # for some reason, we need to add the section_offset of .text here
80
84
  function_offset = code_base_address + symbol.value
81
- if function_offset not in self._func_symbols:
82
- self._func_symbols[function_offset] = function_name
85
+ if function_offset not in function_symbols:
86
+ function_symbols[function_offset] = function_name
87
+ return function_symbols
88
+
89
+ def parseImports(self, lief_binary):
90
+ import_symbols = {}
91
+ for imported_library in lief_binary.imports:
92
+ for func in imported_library.entries:
93
+ if func.name:
94
+ import_symbols[func.iat_address + lief_binary.imagebase] = (imported_library.name.lower(), func.name)
95
+ elif func.is_ordinal:
96
+ resolved_ordinal = OrdinalHelper.resolveOrdinal(imported_library.name.lower(), func.ordinal)
97
+ ordinal_name = resolved_ordinal if resolved_ordinal else "#%s" % func.ordinal
98
+ import_symbols[func.iat_address + lief_binary.imagebase] = (imported_library.name.lower(), ordinal_name)
99
+ return import_symbols
83
100
 
84
101
  def getSymbol(self, address):
85
102
  return self._func_symbols.get(address, "")
@@ -32,8 +32,10 @@ class IdaInterface(object):
32
32
  instance = None
33
33
  def __init__(self):
34
34
  if not IdaInterface.instance:
35
- if idaapi.IDA_SDK_VERSION >= 740:
35
+ if idaapi.IDA_SDK_VERSION >= 740 and idaapi.IDA_SDK_VERSION < 900:
36
36
  IdaInterface.instance = Ida74Interface()
37
+ if idaapi.IDA_SDK_VERSION >= 900:
38
+ IdaInterface.instance = Ida90Interface()
37
39
  else:
38
40
  IdaInterface.instance = Ida73Interface()
39
41
 
@@ -283,3 +285,119 @@ class Ida73Interface(BackendInterface):
283
285
  else:
284
286
  self._api_map[addr] = name
285
287
  return True
288
+
289
+ class Ida90Interface(BackendInterface):
290
+
291
+ def __init__(self):
292
+ self.version = "IDA Pro 9.0"
293
+ self._processor_map = {
294
+ "metapc": "intel"
295
+ }
296
+ self._api_map = {}
297
+ self._import_module_name = ""
298
+
299
+ def getArchitecture(self):
300
+ procname = idaapi.inf_get_procname()
301
+ if procname in self._processor_map:
302
+ return self._processor_map[procname]
303
+ else:
304
+ raise ValueError("Unsupported Architecture")
305
+
306
+ def getBitness(self):
307
+ # https://blog.junron.dev/IDAPython%20Research/IDAPython%208%20to%209.html#idaapi-get-inf-structure
308
+ bits = None
309
+ if idaapi.inf_is_64bit():
310
+ bits = 64
311
+ elif idaapi.inf_is_32bit_exactly():
312
+ bits = 32
313
+ else:
314
+ bits = 16
315
+ return bits
316
+
317
+ def getFunctions(self):
318
+ return sorted([offset for offset in idautils.Functions()])
319
+
320
+ def getBlocks(self, function_offset):
321
+ blocks = []
322
+ function_chart = ida_gdl.FlowChart(ida_funcs.get_func(function_offset))
323
+ for block in function_chart:
324
+ extracted_block = []
325
+ for instruction in idautils.Heads(block.start_ea, block.end_ea):
326
+ if ida_bytes.is_code(ida_bytes.get_flags(instruction)):
327
+ extracted_block.append(instruction)
328
+ if extracted_block:
329
+ blocks.append(extracted_block)
330
+ return sorted(blocks)
331
+
332
+ def getInstructionBytes(self, offset):
333
+ ins = idautils.DecodeInstruction(offset)
334
+ ins_bytes = ida_bytes.get_bytes(offset, ins.size)
335
+ return ins_bytes
336
+
337
+ def getCodeInRefs(self, offset):
338
+ return [(ref_from, offset) for ref_from in idautils.CodeRefsTo(offset, True)]
339
+
340
+ def getCodeOutRefs(self, offset):
341
+ return [(offset, ref_to) for ref_to in idautils.CodeRefsFrom(offset, True)]
342
+
343
+ def getFunctionSymbols(self, demangle=False):
344
+ function_symbols = {}
345
+ function_offsets = self.getFunctions()
346
+ for function_offset in function_offsets:
347
+ function_name = ida_funcs.get_func_name(function_offset)
348
+ # apply demangling if required
349
+ if demangle and "@" in function_name:
350
+ demangled = ida_name.demangle_name(function_name, 0)
351
+ if demangled:
352
+ function_name = demangled
353
+ if not re.match("sub_[0-9a-fA-F]+", function_name):
354
+ function_symbols[function_offset] = function_name
355
+ return function_symbols
356
+
357
+ def getBaseAddr(self):
358
+ base_addr = 0
359
+ segment_starts = [ea for ea in idautils.Segments()]
360
+ if segment_starts:
361
+ first_segment_start = segment_starts[0]
362
+ # re-align by 0x10000 to reflect typically allocation behaviour for IDA-mapped binaries
363
+ first_segment_start = (first_segment_start / 0x10000) * 0x10000
364
+ base_addr = int(first_segment_start)
365
+ return base_addr
366
+
367
+ def getBinary(self):
368
+ result = b""
369
+ segment = ida_segment.get_first_seg()
370
+ while segment:
371
+ result += ida_bytes.get_bytes(segment.start_ea, segment.end_ea - segment.start_ea)
372
+ segment = ida_segment.get_next_seg(segment.end_ea)
373
+ return result
374
+
375
+ def getApiMap(self):
376
+ self._api_map = {}
377
+ num_imports = ida_nalt.get_import_module_qty()
378
+ for i in range(0, num_imports):
379
+ self._import_module_name = ida_nalt.get_import_module_name(i)
380
+ ida_nalt.enum_import_names(i, self._cbEnumImports)
381
+ return self._api_map
382
+
383
+ def isExternalFunction(self, function_offset):
384
+ function_segment = ida_segment.getseg(function_offset)
385
+ function_segment_name = ida_segment.get_segm_name(function_segment)
386
+ is_extern = function_segment_name in ["extern", "UNDEF"]
387
+ return is_extern
388
+
389
+ def makeFunction(self, instruction):
390
+ return ida_funcs.add_func(instruction)
391
+
392
+ def makeNameEx(self, address, name, warning_level=None):
393
+ if warning_level is None:
394
+ warning_level=idc.SN_NOWARN
395
+ return idc.set_name(address, name, warning_level)
396
+
397
+ def _cbEnumImports(self, addr, name, ordinal):
398
+ # potentially use: idc.Name(addr)
399
+ if self._import_module_name:
400
+ self._api_map[addr] = self._import_module_name + "!" + name
401
+ else:
402
+ self._api_map[addr] = name
403
+ return True