memdiver 0.5.6__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (232) hide show
  1. memdiver/__init__.py +8 -0
  2. memdiver/algorithms/__init__.py +1 -0
  3. memdiver/algorithms/base.py +62 -0
  4. memdiver/algorithms/known_key/__init__.py +1 -0
  5. memdiver/algorithms/known_key/exact_match.py +47 -0
  6. memdiver/algorithms/patterns/__init__.py +1 -0
  7. memdiver/algorithms/patterns/boringssl_tls13.json +28 -0
  8. memdiver/algorithms/patterns/openssh_ssh2.json +28 -0
  9. memdiver/algorithms/patterns/openssl_tls12.json +28 -0
  10. memdiver/algorithms/registry.py +74 -0
  11. memdiver/algorithms/unknown_key/__init__.py +1 -0
  12. memdiver/algorithms/unknown_key/change_point.py +223 -0
  13. memdiver/algorithms/unknown_key/constraint_validator.py +248 -0
  14. memdiver/algorithms/unknown_key/differential.py +238 -0
  15. memdiver/algorithms/unknown_key/entropy_scan.py +83 -0
  16. memdiver/algorithms/unknown_key/pattern_match.py +124 -0
  17. memdiver/algorithms/unknown_key/structure_scan.py +126 -0
  18. memdiver/algorithms/unknown_key/user_regex.py +62 -0
  19. memdiver/api/__init__.py +0 -0
  20. memdiver/api/config.py +84 -0
  21. memdiver/api/dependencies.py +48 -0
  22. memdiver/api/main.py +177 -0
  23. memdiver/api/models.py +111 -0
  24. memdiver/api/routers/__init__.py +0 -0
  25. memdiver/api/routers/analysis.py +372 -0
  26. memdiver/api/routers/architect.py +131 -0
  27. memdiver/api/routers/consensus.py +182 -0
  28. memdiver/api/routers/dataset.py +160 -0
  29. memdiver/api/routers/dumps.py +58 -0
  30. memdiver/api/routers/experiment.py +101 -0
  31. memdiver/api/routers/inspect.py +600 -0
  32. memdiver/api/routers/oracles.py +142 -0
  33. memdiver/api/routers/path.py +141 -0
  34. memdiver/api/routers/pipeline.py +419 -0
  35. memdiver/api/routers/sessions.py +107 -0
  36. memdiver/api/routers/structures.py +122 -0
  37. memdiver/api/routers/tasks.py +64 -0
  38. memdiver/api/services/__init__.py +0 -0
  39. memdiver/api/services/analysis_service.py +312 -0
  40. memdiver/api/services/artifact_store.py +288 -0
  41. memdiver/api/services/consensus_session.py +141 -0
  42. memdiver/api/services/oracle_registry.py +417 -0
  43. memdiver/api/services/progress_bus.py +189 -0
  44. memdiver/api/services/reader_cache.py +337 -0
  45. memdiver/api/services/session_service.py +86 -0
  46. memdiver/api/services/task_manager.py +666 -0
  47. memdiver/api/ws/__init__.py +0 -0
  48. memdiver/api/ws/progress.py +94 -0
  49. memdiver/architect/__init__.py +0 -0
  50. memdiver/architect/json_exporter.py +71 -0
  51. memdiver/architect/pattern_generator.py +171 -0
  52. memdiver/architect/static_checker.py +73 -0
  53. memdiver/architect/volatility3_exporter.py +323 -0
  54. memdiver/architect/yara_exporter.py +70 -0
  55. memdiver/cli.py +1204 -0
  56. memdiver/config.json +18 -0
  57. memdiver/core/__init__.py +8 -0
  58. memdiver/core/alignment_filter.py +67 -0
  59. memdiver/core/binary_formats/__init__.py +1 -0
  60. memdiver/core/binary_formats/elf_core_reader.py +361 -0
  61. memdiver/core/binary_formats/elf_defs.py +34 -0
  62. memdiver/core/binary_formats/kaitai_adapter.py +208 -0
  63. memdiver/core/binary_formats/kaitai_compiled/__init__.py +1 -0
  64. memdiver/core/binary_formats/kaitai_compiled/elf.py +329 -0
  65. memdiver/core/binary_formats/kaitai_compiled/mach_o.py +316 -0
  66. memdiver/core/binary_formats/kaitai_compiled/microsoft_pe.py +339 -0
  67. memdiver/core/binary_formats/kaitai_compiled/msl.py +197 -0
  68. memdiver/core/binary_formats/kaitai_registry.py +143 -0
  69. memdiver/core/binary_formats/macho_defs.py +30 -0
  70. memdiver/core/binary_formats/navigator.py +227 -0
  71. memdiver/core/binary_formats/pe_defs.py +42 -0
  72. memdiver/core/classify.py +40 -0
  73. memdiver/core/config_schema.py +113 -0
  74. memdiver/core/constants.py +39 -0
  75. memdiver/core/dataset_metadata.py +186 -0
  76. memdiver/core/discovery.py +468 -0
  77. memdiver/core/display_labels.py +23 -0
  78. memdiver/core/dump_driver.py +204 -0
  79. memdiver/core/dump_io.py +108 -0
  80. memdiver/core/dump_search.py +70 -0
  81. memdiver/core/dump_source.py +364 -0
  82. memdiver/core/entropy.py +168 -0
  83. memdiver/core/format_detect.py +136 -0
  84. memdiver/core/input_schemas.py +62 -0
  85. memdiver/core/kdf.py +196 -0
  86. memdiver/core/kdf_base.py +74 -0
  87. memdiver/core/kdf_registry.py +80 -0
  88. memdiver/core/kdf_ssh.py +176 -0
  89. memdiver/core/kdf_tls.py +181 -0
  90. memdiver/core/keylog.py +87 -0
  91. memdiver/core/keylog_templates.py +108 -0
  92. memdiver/core/log.py +49 -0
  93. memdiver/core/models.py +153 -0
  94. memdiver/core/msl_helpers.py +26 -0
  95. memdiver/core/phase_normalizer.py +147 -0
  96. memdiver/core/proc_maps_parser.py +113 -0
  97. memdiver/core/protocols.py +150 -0
  98. memdiver/core/region_align.py +218 -0
  99. memdiver/core/region_analysis.py +122 -0
  100. memdiver/core/strings.py +143 -0
  101. memdiver/core/structure_defs.py +157 -0
  102. memdiver/core/structure_library.py +105 -0
  103. memdiver/core/structure_library_ssh.py +42 -0
  104. memdiver/core/structure_library_tls.py +169 -0
  105. memdiver/core/structure_loader.py +43 -0
  106. memdiver/core/structure_overlay.py +241 -0
  107. memdiver/core/structure_schema.py +119 -0
  108. memdiver/core/variance.py +195 -0
  109. memdiver/engine/__init__.py +0 -0
  110. memdiver/engine/batch.py +196 -0
  111. memdiver/engine/batch_task_runner.py +210 -0
  112. memdiver/engine/brute_force.py +446 -0
  113. memdiver/engine/cancellation.py +36 -0
  114. memdiver/engine/candidate_pipeline.py +337 -0
  115. memdiver/engine/consensus.py +329 -0
  116. memdiver/engine/consensus_msl.py +182 -0
  117. memdiver/engine/convergence.py +181 -0
  118. memdiver/engine/correlator.py +179 -0
  119. memdiver/engine/derived_keys.py +70 -0
  120. memdiver/engine/diff_store.py +195 -0
  121. memdiver/engine/experiment_task_runner.py +458 -0
  122. memdiver/engine/nsweep.py +324 -0
  123. memdiver/engine/oracle.py +145 -0
  124. memdiver/engine/pipeline.py +250 -0
  125. memdiver/engine/pipeline_runner.py +586 -0
  126. memdiver/engine/progress.py +104 -0
  127. memdiver/engine/project_db.py +193 -0
  128. memdiver/engine/results.py +79 -0
  129. memdiver/engine/serializer.py +165 -0
  130. memdiver/engine/session_store.py +234 -0
  131. memdiver/engine/verification.py +131 -0
  132. memdiver/engine/vol3_emit.py +245 -0
  133. memdiver/engine/worker.py +57 -0
  134. memdiver/frontend/dist/assets/EntropyChart-8M8HrU-B.js +1 -0
  135. memdiver/frontend/dist/assets/EntropyChart-DAtoAJCY.js +1 -0
  136. memdiver/frontend/dist/assets/SurvivorCurve-BY20wrM1.js +1 -0
  137. memdiver/frontend/dist/assets/SurvivorCurve-D9JxV81S.js +1 -0
  138. memdiver/frontend/dist/assets/analysis-store-BpQfEZUW.js +1 -0
  139. memdiver/frontend/dist/assets/client-CNl5BiOS.js +1 -0
  140. memdiver/frontend/dist/assets/dump-store-Dwu_Tzhp.js +1 -0
  141. memdiver/frontend/dist/assets/hex-store-CpkAyGnC.js +1 -0
  142. memdiver/frontend/dist/assets/hex-store-gtC9qvXO.js +1 -0
  143. memdiver/frontend/dist/assets/index-CkDrSf6P.js +87 -0
  144. memdiver/frontend/dist/assets/index-PoxQeNeU.css +2 -0
  145. memdiver/frontend/dist/assets/jsx-runtime-sPcSWLTJ.js +1 -0
  146. memdiver/frontend/dist/assets/normalize-plot-twAkls4V.js +4030 -0
  147. memdiver/frontend/dist/assets/primitives-CAn_uvh6.js +1 -0
  148. memdiver/frontend/dist/assets/react-BcFANYnC.js +1 -0
  149. memdiver/frontend/dist/assets/react-CYzKIDNi.js +1 -0
  150. memdiver/frontend/dist/assets/results-store-BSf3LrQV.js +1 -0
  151. memdiver/frontend/dist/assets/strings-store-CicX0gXn.js +1 -0
  152. memdiver/frontend/dist/assets/survivor-traces-DyBxrQCs.js +1 -0
  153. memdiver/frontend/dist/assets/useChartTheme-7Z3828aq.js +1 -0
  154. memdiver/frontend/dist/favicon.svg +1 -0
  155. memdiver/frontend/dist/icons.svg +24 -0
  156. memdiver/frontend/dist/index.html +30 -0
  157. memdiver/frontend/dist/memdiver-logo.svg +352 -0
  158. memdiver/harvester/__init__.py +0 -0
  159. memdiver/harvester/ingestor.py +90 -0
  160. memdiver/harvester/metadata_store.py +83 -0
  161. memdiver/harvester/sidecar.py +78 -0
  162. memdiver/legacy_app.py +145 -0
  163. memdiver/mcp_server/__init__.py +12 -0
  164. memdiver/mcp_server/server.py +208 -0
  165. memdiver/mcp_server/session.py +67 -0
  166. memdiver/mcp_server/tools.py +125 -0
  167. memdiver/mcp_server/tools_inspect.py +352 -0
  168. memdiver/mcp_server/tools_pipeline.py +226 -0
  169. memdiver/mcp_server/tools_xref.py +86 -0
  170. memdiver/msl/__init__.py +53 -0
  171. memdiver/msl/block_iter.py +57 -0
  172. memdiver/msl/block_tree.py +81 -0
  173. memdiver/msl/compress.py +73 -0
  174. memdiver/msl/decoders.py +590 -0
  175. memdiver/msl/decoders_ext.py +287 -0
  176. memdiver/msl/enums.py +184 -0
  177. memdiver/msl/hashing.py +45 -0
  178. memdiver/msl/importer.py +122 -0
  179. memdiver/msl/integrity.py +72 -0
  180. memdiver/msl/key_extract.py +168 -0
  181. memdiver/msl/page_map.py +135 -0
  182. memdiver/msl/reader.py +256 -0
  183. memdiver/msl/session_extract.py +156 -0
  184. memdiver/msl/string_extract.py +139 -0
  185. memdiver/msl/types.py +387 -0
  186. memdiver/msl/writer.py +217 -0
  187. memdiver/msl/xref_resolver.py +125 -0
  188. memdiver/run.py +874 -0
  189. memdiver/ui/__init__.py +0 -0
  190. memdiver/ui/adapters.py +56 -0
  191. memdiver/ui/components/__init__.py +0 -0
  192. memdiver/ui/components/bookmark_store.py +70 -0
  193. memdiver/ui/components/color_scheme.py +70 -0
  194. memdiver/ui/components/header.py +67 -0
  195. memdiver/ui/components/hex_pager.py +52 -0
  196. memdiver/ui/components/hex_renderer.py +117 -0
  197. memdiver/ui/components/html_builder.py +89 -0
  198. memdiver/ui/components/progress_helper.py +54 -0
  199. memdiver/ui/components/setup_wizard.py +158 -0
  200. memdiver/ui/controls/__init__.py +0 -0
  201. memdiver/ui/controls/analysis_panel.py +139 -0
  202. memdiver/ui/controls/config_panel.py +101 -0
  203. memdiver/ui/controls/selector_panel.py +169 -0
  204. memdiver/ui/locales.py +37 -0
  205. memdiver/ui/mode.py +107 -0
  206. memdiver/ui/nicegui/__init__.py +1 -0
  207. memdiver/ui/nicegui/api_routes.py +74 -0
  208. memdiver/ui/nicegui/theme.py +74 -0
  209. memdiver/ui/nicegui/wizard.py +104 -0
  210. memdiver/ui/nicegui/workspace.py +84 -0
  211. memdiver/ui/state.py +137 -0
  212. memdiver/ui/views/__init__.py +0 -0
  213. memdiver/ui/views/block_navigator.py +288 -0
  214. memdiver/ui/views/consensus_view.py +71 -0
  215. memdiver/ui/views/cross_library.py +69 -0
  216. memdiver/ui/views/differential_diff.py +90 -0
  217. memdiver/ui/views/entropy_chart.py +82 -0
  218. memdiver/ui/views/heatmap.py +97 -0
  219. memdiver/ui/views/hex_comparison.py +116 -0
  220. memdiver/ui/views/hex_navigator.py +171 -0
  221. memdiver/ui/views/hex_viewer.py +125 -0
  222. memdiver/ui/views/investigation_panel.py +122 -0
  223. memdiver/ui/views/phase_lifecycle.py +76 -0
  224. memdiver/ui/views/session_view.py +135 -0
  225. memdiver/ui/views/variance_map.py +80 -0
  226. memdiver/ui/views/vas_view.py +185 -0
  227. memdiver-0.5.6.dist-info/METADATA +234 -0
  228. memdiver-0.5.6.dist-info/RECORD +232 -0
  229. memdiver-0.5.6.dist-info/WHEEL +5 -0
  230. memdiver-0.5.6.dist-info/entry_points.txt +2 -0
  231. memdiver-0.5.6.dist-info/licenses/LICENSE +201 -0
  232. memdiver-0.5.6.dist-info/top_level.txt +1 -0
memdiver/__init__.py ADDED
@@ -0,0 +1,8 @@
1
+ """MemDiver - Interactive platform for identifying and analyzing data structures in memory dumps."""
2
+
3
+ from importlib.metadata import PackageNotFoundError, version as _version
4
+
5
+ try:
6
+ __version__ = _version("memdiver")
7
+ except PackageNotFoundError:
8
+ __version__ = "0.0.0+unknown"
@@ -0,0 +1 @@
1
+ """Algorithm plugin system for memory dump analysis."""
@@ -0,0 +1,62 @@
1
+ """Base classes for analysis algorithms."""
2
+
3
+ from abc import ABC, abstractmethod
4
+ from dataclasses import dataclass, field
5
+ from typing import Any, Dict, List, Optional
6
+
7
+ from core.models import CryptoSecret, deprecated_kwarg
8
+
9
+
10
+ @dataclass
11
+ class AnalysisContext:
12
+ """Context provided to algorithms."""
13
+ library: str
14
+ protocol_version: str
15
+ phase: str
16
+ secrets: List[CryptoSecret] = field(default_factory=list)
17
+ extra: Dict[str, Any] = field(default_factory=dict)
18
+
19
+ @property
20
+ def tls_version(self) -> str:
21
+ """Backward-compatible alias for protocol_version."""
22
+ return self.protocol_version
23
+
24
+ @tls_version.setter
25
+ def tls_version(self, value: str) -> None:
26
+ self.protocol_version = value
27
+
28
+
29
+ deprecated_kwarg(AnalysisContext, "tls_version", "protocol_version")
30
+
31
+
32
+ @dataclass
33
+ class Match:
34
+ """A single match found by an algorithm."""
35
+ offset: int
36
+ length: int
37
+ confidence: float
38
+ label: str = ""
39
+ data: bytes = b""
40
+ metadata: Dict[str, Any] = field(default_factory=dict)
41
+
42
+
43
+ @dataclass
44
+ class AlgorithmResult:
45
+ """Result from running an algorithm."""
46
+ algorithm_name: str
47
+ confidence: float
48
+ matches: List[Match] = field(default_factory=list)
49
+ metadata: Dict[str, Any] = field(default_factory=dict)
50
+
51
+
52
+ class BaseAlgorithm(ABC):
53
+ """Base class for all analysis algorithms."""
54
+
55
+ name: str = ""
56
+ description: str = ""
57
+ mode: str = "" # Use KNOWN_KEY or UNKNOWN_KEY from core.constants
58
+
59
+ @abstractmethod
60
+ def run(self, dump_data: bytes, context: AnalysisContext) -> AlgorithmResult:
61
+ """Run the algorithm on dump data."""
62
+ ...
@@ -0,0 +1 @@
1
+ """Known-key algorithms - require ground truth secrets."""
@@ -0,0 +1,47 @@
1
+ """Exact byte match algorithm - search for known secret bytes in dump."""
2
+
3
+ from algorithms.base import AlgorithmResult, AnalysisContext, BaseAlgorithm, Match
4
+ from core.constants import KNOWN_KEY
5
+
6
+
7
+ class ExactMatchAlgorithm(BaseAlgorithm):
8
+ """Search for exact secret byte sequences in memory dumps."""
9
+
10
+ name = "exact_match"
11
+ description = "Search for exact secret byte sequences"
12
+ mode = KNOWN_KEY
13
+
14
+ def run(self, dump_data: bytes, context: AnalysisContext) -> AlgorithmResult:
15
+ matches = []
16
+
17
+ for secret in context.secrets:
18
+ needle = secret.secret_value
19
+ start = 0
20
+ while True:
21
+ idx = dump_data.find(needle, start)
22
+ if idx == -1:
23
+ break
24
+ matches.append(Match(
25
+ offset=idx,
26
+ length=len(needle),
27
+ confidence=1.0,
28
+ label=secret.secret_type,
29
+ data=needle,
30
+ metadata={"secret_type": secret.secret_type},
31
+ ))
32
+ start = idx + 1
33
+
34
+ found_types = set(m.label for m in matches)
35
+ total_types = len(set(s.secret_type for s in context.secrets))
36
+ confidence = len(found_types) / total_types if total_types > 0 else 0.0
37
+
38
+ return AlgorithmResult(
39
+ algorithm_name=self.name,
40
+ confidence=confidence,
41
+ matches=matches,
42
+ metadata={
43
+ "total_secret_types": total_types,
44
+ "found_secret_types": len(found_types),
45
+ "found_types_list": sorted(found_types),
46
+ },
47
+ )
@@ -0,0 +1 @@
1
+ """Pattern-based algorithms for structural matching."""
@@ -0,0 +1,28 @@
1
+ {
2
+ "name": "boringssl_tls13_exporter_secret",
3
+ "description": "Structural pattern for BoringSSL TLS 1.3 EXPORTER_SECRET in memory",
4
+ "applicable_to": {
5
+ "libraries": ["boringssl"],
6
+ "protocol_versions": ["13"]
7
+ },
8
+ "key_spec": {
9
+ "length": 32,
10
+ "entropy_min": 7.0
11
+ },
12
+ "pattern": {
13
+ "before": [
14
+ {
15
+ "offset": -8,
16
+ "bytes": "00 00 00 00 00 00 00 20",
17
+ "mask": "00 00 00 00 00 00 00 ff"
18
+ }
19
+ ],
20
+ "after": [
21
+ {
22
+ "offset": 32,
23
+ "bytes": "00 00 00 00 00 00 00 00",
24
+ "mask": "00 00 00 00 00 00 00 00"
25
+ }
26
+ ]
27
+ }
28
+ }
@@ -0,0 +1,28 @@
1
+ {
2
+ "name": "openssh_ssh2_session_key",
3
+ "description": "Structural pattern for OpenSSH SSH-2 session key in memory",
4
+ "applicable_to": {
5
+ "libraries": ["openssh"],
6
+ "protocol_versions": ["2"]
7
+ },
8
+ "key_spec": {
9
+ "length": 32,
10
+ "entropy_min": 7.0
11
+ },
12
+ "pattern": {
13
+ "before": [
14
+ {
15
+ "offset": -4,
16
+ "bytes": "00 00 00 20",
17
+ "mask": "00 00 00 ff"
18
+ }
19
+ ],
20
+ "after": [
21
+ {
22
+ "offset": 32,
23
+ "bytes": "00 00 00 00",
24
+ "mask": "00 00 00 00"
25
+ }
26
+ ]
27
+ }
28
+ }
@@ -0,0 +1,28 @@
1
+ {
2
+ "name": "openssl_tls12_master_secret",
3
+ "description": "Structural pattern for OpenSSL TLS 1.2 master secret in memory",
4
+ "applicable_to": {
5
+ "libraries": ["openssl"],
6
+ "protocol_versions": ["12"]
7
+ },
8
+ "key_spec": {
9
+ "length": 48,
10
+ "entropy_min": 7.0
11
+ },
12
+ "pattern": {
13
+ "before": [
14
+ {
15
+ "offset": -4,
16
+ "bytes": "00 00 00 30",
17
+ "mask": "00 00 00 ff"
18
+ }
19
+ ],
20
+ "after": [
21
+ {
22
+ "offset": 48,
23
+ "bytes": "00 00 00 00 00 00 00 00",
24
+ "mask": "00 00 00 00 00 00 00 00"
25
+ }
26
+ ]
27
+ }
28
+ }
@@ -0,0 +1,74 @@
1
+ """Auto-discovery registry for algorithm plugins."""
2
+
3
+ import importlib
4
+ import pkgutil
5
+ from pathlib import Path
6
+ from typing import Dict, List
7
+
8
+ from core.constants import AlgorithmMode
9
+
10
+ from .base import BaseAlgorithm
11
+
12
+
13
+ class AlgorithmRegistry:
14
+ """Discover and manage algorithm plugins."""
15
+
16
+ def __init__(self):
17
+ self._algorithms: Dict[str, BaseAlgorithm] = {}
18
+
19
+ def discover(self) -> None:
20
+ """Walk known_key/ and unknown_key/ subdirectories to find algorithms."""
21
+ base_dir = Path(__file__).parent
22
+
23
+ for subdir in ["known_key", "unknown_key", "patterns"]:
24
+ pkg_path = base_dir / subdir
25
+ if not pkg_path.is_dir():
26
+ continue
27
+
28
+ pkg_name = f"algorithms.{subdir}"
29
+ try:
30
+ pkg = importlib.import_module(pkg_name)
31
+ except ImportError:
32
+ continue
33
+
34
+ for importer, modname, ispkg in pkgutil.walk_packages(
35
+ path=[str(pkg_path)], prefix=f"{pkg_name}."
36
+ ):
37
+ try:
38
+ mod = importlib.import_module(modname)
39
+ except ImportError:
40
+ continue
41
+
42
+ for attr_name in dir(mod):
43
+ attr = getattr(mod, attr_name)
44
+ if (isinstance(attr, type)
45
+ and issubclass(attr, BaseAlgorithm)
46
+ and attr is not BaseAlgorithm
47
+ and hasattr(attr, 'name')
48
+ and attr.name):
49
+ instance = attr()
50
+ self._algorithms[instance.name] = instance
51
+
52
+ def get(self, name: str) -> BaseAlgorithm:
53
+ return self._algorithms[name]
54
+
55
+ def list_all(self) -> List[BaseAlgorithm]:
56
+ return list(self._algorithms.values())
57
+
58
+ def list_by_mode(self, mode: AlgorithmMode) -> List[BaseAlgorithm]:
59
+ return [a for a in self._algorithms.values() if a.mode == mode]
60
+
61
+ @property
62
+ def names(self) -> List[str]:
63
+ return sorted(self._algorithms.keys())
64
+
65
+
66
+ _registry = None
67
+
68
+
69
+ def get_registry() -> AlgorithmRegistry:
70
+ global _registry
71
+ if _registry is None:
72
+ _registry = AlgorithmRegistry()
73
+ _registry.discover()
74
+ return _registry
@@ -0,0 +1 @@
1
+ """Unknown-key algorithms - detect keys without ground truth."""
@@ -0,0 +1,223 @@
1
+ """Entropy change-point detection using CUSUM for finding key-material plateaus."""
2
+
3
+ from bisect import bisect_left, bisect_right
4
+ from typing import Dict, List, Tuple
5
+
6
+ from algorithms.base import AlgorithmResult, AnalysisContext, BaseAlgorithm, Match
7
+ from core.constants import UNKNOWN_KEY
8
+ from core.entropy import compute_entropy_profile
9
+
10
+
11
+ class ChangePointAlgorithm(BaseAlgorithm):
12
+ """Detect key-material plateaus via CUSUM entropy change-point analysis.
13
+
14
+ Three stages:
15
+ 1. Entropy profiling via sliding-window Shannon entropy.
16
+ 2. Gradient CUSUM edge detection on the first difference of the profile.
17
+ 3. Plateau extraction by pairing rising/falling edges that match
18
+ expected key widths and exceed the entropy threshold.
19
+ """
20
+
21
+ name = "change_point"
22
+ description = "Entropy change-point detection using CUSUM to find key-material plateaus"
23
+ mode = UNKNOWN_KEY
24
+
25
+ DEFAULT_WINDOW = 32
26
+ DEFAULT_ENTROPY_THRESHOLD = 4.5
27
+ DEFAULT_CUSUM_THRESHOLD = 0.8
28
+ DEFAULT_DRIFT = 0.05
29
+ DEFAULT_STEP = 16
30
+ DEFAULT_PLATEAU_WIDTHS = [32, 48]
31
+
32
+ @staticmethod
33
+ def _cusum_change_points(
34
+ profile: List[Tuple[int, float]],
35
+ threshold: float = 0.8,
36
+ drift: float = 0.05,
37
+ ) -> List[Tuple[int, str, float]]:
38
+ """Detect change points using gradient CUSUM.
39
+
40
+ Reports the onset offset (not detection offset) for each edge.
41
+ Returns list of (offset, direction, cusum_value) tuples where
42
+ direction is "up" (rising) or "down" (falling).
43
+ """
44
+ if len(profile) < 2:
45
+ return []
46
+
47
+ s_high = 0.0
48
+ s_low = 0.0
49
+ change_points: List[Tuple[int, str, float]] = []
50
+
51
+ # Track where the current accumulation run started so we can
52
+ # report the onset offset rather than the detection offset.
53
+ run_start_high = profile[0][0]
54
+ run_start_low = profile[0][0]
55
+
56
+ for i in range(1, len(profile)):
57
+ offset = profile[i][0]
58
+ gradient = profile[i][1] - profile[i - 1][1]
59
+
60
+ # Track run-start: reset when the statistic was at zero.
61
+ prev_s_high = s_high
62
+ prev_s_low = s_low
63
+
64
+ s_high = max(0.0, s_high + gradient - drift)
65
+ s_low = max(0.0, s_low - gradient - drift)
66
+
67
+ if prev_s_high == 0.0 and s_high > 0.0:
68
+ run_start_high = offset
69
+ if prev_s_low == 0.0 and s_low > 0.0:
70
+ run_start_low = offset
71
+
72
+ if s_high > threshold:
73
+ change_points.append((run_start_high, "up", s_high))
74
+ s_high = 0.0
75
+
76
+ if s_low > threshold:
77
+ change_points.append((run_start_low, "down", s_low))
78
+ s_low = 0.0
79
+
80
+ return change_points
81
+
82
+ def _extract_plateaus(
83
+ self,
84
+ dump_data: bytes,
85
+ profile: List[Tuple[int, float]],
86
+ change_points: List[Tuple[int, str, float]],
87
+ entropy_threshold: float,
88
+ plateau_widths: List[int],
89
+ window: int,
90
+ ) -> List[Match]:
91
+ """Pair rising/falling edges into key-sized high-entropy plateaus.
92
+
93
+ Accepts pairs whose span matches a target key width (adjusted for
94
+ window smearing) and whose mean entropy exceeds the threshold.
95
+ """
96
+ # Build an offset -> entropy lookup for fast slicing.
97
+ entropy_lookup: Dict[int, float] = {off: ent for off, ent in profile}
98
+ sorted_offsets = sorted(entropy_lookup.keys())
99
+
100
+ # Pair consecutive up -> down transitions.
101
+ pairs: List[Tuple[int, int, float, float]] = []
102
+ i = 0
103
+ while i < len(change_points) - 1:
104
+ offset_up, direction_up, strength_up = change_points[i]
105
+ offset_down, direction_down, strength_down = change_points[i + 1]
106
+
107
+ if direction_up == "up" and direction_down == "down":
108
+ pairs.append((offset_up, offset_down, strength_up, strength_down))
109
+ i += 2
110
+ else:
111
+ i += 1
112
+
113
+ width_tolerance = 16
114
+ matches: List[Match] = []
115
+
116
+ for start_off, end_off, strength_up, strength_down in pairs:
117
+ profile_span = end_off - start_off
118
+ if profile_span <= 0:
119
+ continue
120
+
121
+ best_key_width = 0
122
+ best_distance = width_tolerance + 1
123
+ for target_w in plateau_widths:
124
+ expected_spans = [target_w, target_w + window - 1]
125
+ for expected in expected_spans:
126
+ dist = abs(profile_span - expected)
127
+ if dist <= width_tolerance and dist < best_distance:
128
+ best_distance = dist
129
+ best_key_width = target_w
130
+
131
+ if best_key_width == 0:
132
+ continue
133
+
134
+ # Compute mean entropy over the plateau region (binary search).
135
+ lo = bisect_left(sorted_offsets, start_off)
136
+ hi = bisect_right(sorted_offsets, end_off)
137
+ segment_offsets = sorted_offsets[lo:hi]
138
+ if not segment_offsets:
139
+ continue
140
+
141
+ segment_entropies = [entropy_lookup[off] for off in segment_offsets]
142
+ mean_entropy = sum(segment_entropies) / len(segment_entropies)
143
+
144
+ if mean_entropy < entropy_threshold:
145
+ continue
146
+
147
+ # The key material starts at the rising edge. Extract
148
+ # best_key_width bytes from that point.
149
+ key_start = start_off
150
+ key_len = min(best_key_width, len(dump_data) - key_start)
151
+ plateau_data = dump_data[key_start:key_start + key_len]
152
+
153
+ cusum_strength = (strength_up + strength_down) / 2.0
154
+ confidence = min(mean_entropy / 8.0, 1.0) * min(cusum_strength / 3.0, 1.0)
155
+
156
+ matches.append(
157
+ Match(
158
+ offset=key_start,
159
+ length=key_len,
160
+ confidence=round(confidence, 4),
161
+ label=f"cusum_plateau_{key_len}B",
162
+ data=plateau_data,
163
+ metadata={
164
+ "entropy_mean": round(mean_entropy, 4),
165
+ "plateau_width": profile_span,
166
+ "estimated_key_width": best_key_width,
167
+ "cusum_strength": round(cusum_strength, 4),
168
+ "profile_segment": [
169
+ round(e, 4) for e in segment_entropies[:10]
170
+ ],
171
+ },
172
+ )
173
+ )
174
+
175
+ return matches
176
+
177
+ def run(self, dump_data: bytes, context: AnalysisContext) -> AlgorithmResult:
178
+ """Run CUSUM change-point detection. Parameters from context.extra."""
179
+ window = context.extra.get("window", self.DEFAULT_WINDOW)
180
+ step = context.extra.get("step", self.DEFAULT_STEP)
181
+ entropy_threshold = context.extra.get(
182
+ "entropy_threshold", self.DEFAULT_ENTROPY_THRESHOLD
183
+ )
184
+ cusum_threshold = context.extra.get(
185
+ "cusum_threshold", self.DEFAULT_CUSUM_THRESHOLD
186
+ )
187
+ drift = context.extra.get("drift", self.DEFAULT_DRIFT)
188
+ plateau_widths = context.extra.get(
189
+ "plateau_widths", self.DEFAULT_PLATEAU_WIDTHS
190
+ )
191
+
192
+ # Stage 1: compute entropy profile.
193
+ profile = compute_entropy_profile(dump_data, window=window, step=step)
194
+
195
+ # Stage 2: detect change points via gradient CUSUM.
196
+ change_points = self._cusum_change_points(
197
+ profile, threshold=cusum_threshold, drift=drift
198
+ )
199
+
200
+ # Stage 3: extract qualifying plateaus.
201
+ matches = self._extract_plateaus(
202
+ dump_data, profile, change_points, entropy_threshold, plateau_widths,
203
+ window,
204
+ )
205
+
206
+ # Overall confidence scales with the number of plausible matches.
207
+ overall_confidence = min(len(matches) / 10.0, 1.0) if matches else 0.0
208
+
209
+ return AlgorithmResult(
210
+ algorithm_name=self.name,
211
+ confidence=round(overall_confidence, 4),
212
+ matches=matches,
213
+ metadata={
214
+ "window": window,
215
+ "step": step,
216
+ "entropy_threshold": entropy_threshold,
217
+ "cusum_threshold": cusum_threshold,
218
+ "drift": drift,
219
+ "plateau_widths": plateau_widths,
220
+ "change_points_detected": len(change_points),
221
+ "plateaus_matched": len(matches),
222
+ },
223
+ )