memdiver 0.5.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- memdiver/__init__.py +8 -0
- memdiver/algorithms/__init__.py +1 -0
- memdiver/algorithms/base.py +62 -0
- memdiver/algorithms/known_key/__init__.py +1 -0
- memdiver/algorithms/known_key/exact_match.py +47 -0
- memdiver/algorithms/patterns/__init__.py +1 -0
- memdiver/algorithms/patterns/boringssl_tls13.json +28 -0
- memdiver/algorithms/patterns/openssh_ssh2.json +28 -0
- memdiver/algorithms/patterns/openssl_tls12.json +28 -0
- memdiver/algorithms/registry.py +74 -0
- memdiver/algorithms/unknown_key/__init__.py +1 -0
- memdiver/algorithms/unknown_key/change_point.py +223 -0
- memdiver/algorithms/unknown_key/constraint_validator.py +248 -0
- memdiver/algorithms/unknown_key/differential.py +238 -0
- memdiver/algorithms/unknown_key/entropy_scan.py +83 -0
- memdiver/algorithms/unknown_key/pattern_match.py +124 -0
- memdiver/algorithms/unknown_key/structure_scan.py +126 -0
- memdiver/algorithms/unknown_key/user_regex.py +62 -0
- memdiver/api/__init__.py +0 -0
- memdiver/api/config.py +84 -0
- memdiver/api/dependencies.py +48 -0
- memdiver/api/main.py +177 -0
- memdiver/api/models.py +111 -0
- memdiver/api/routers/__init__.py +0 -0
- memdiver/api/routers/analysis.py +372 -0
- memdiver/api/routers/architect.py +131 -0
- memdiver/api/routers/consensus.py +182 -0
- memdiver/api/routers/dataset.py +160 -0
- memdiver/api/routers/dumps.py +58 -0
- memdiver/api/routers/experiment.py +101 -0
- memdiver/api/routers/inspect.py +600 -0
- memdiver/api/routers/oracles.py +142 -0
- memdiver/api/routers/path.py +141 -0
- memdiver/api/routers/pipeline.py +419 -0
- memdiver/api/routers/sessions.py +107 -0
- memdiver/api/routers/structures.py +122 -0
- memdiver/api/routers/tasks.py +64 -0
- memdiver/api/services/__init__.py +0 -0
- memdiver/api/services/analysis_service.py +312 -0
- memdiver/api/services/artifact_store.py +288 -0
- memdiver/api/services/consensus_session.py +141 -0
- memdiver/api/services/oracle_registry.py +417 -0
- memdiver/api/services/progress_bus.py +189 -0
- memdiver/api/services/reader_cache.py +337 -0
- memdiver/api/services/session_service.py +86 -0
- memdiver/api/services/task_manager.py +666 -0
- memdiver/api/ws/__init__.py +0 -0
- memdiver/api/ws/progress.py +94 -0
- memdiver/architect/__init__.py +0 -0
- memdiver/architect/json_exporter.py +71 -0
- memdiver/architect/pattern_generator.py +171 -0
- memdiver/architect/static_checker.py +73 -0
- memdiver/architect/volatility3_exporter.py +323 -0
- memdiver/architect/yara_exporter.py +70 -0
- memdiver/cli.py +1204 -0
- memdiver/config.json +18 -0
- memdiver/core/__init__.py +8 -0
- memdiver/core/alignment_filter.py +67 -0
- memdiver/core/binary_formats/__init__.py +1 -0
- memdiver/core/binary_formats/elf_core_reader.py +361 -0
- memdiver/core/binary_formats/elf_defs.py +34 -0
- memdiver/core/binary_formats/kaitai_adapter.py +208 -0
- memdiver/core/binary_formats/kaitai_compiled/__init__.py +1 -0
- memdiver/core/binary_formats/kaitai_compiled/elf.py +329 -0
- memdiver/core/binary_formats/kaitai_compiled/mach_o.py +316 -0
- memdiver/core/binary_formats/kaitai_compiled/microsoft_pe.py +339 -0
- memdiver/core/binary_formats/kaitai_compiled/msl.py +197 -0
- memdiver/core/binary_formats/kaitai_registry.py +143 -0
- memdiver/core/binary_formats/macho_defs.py +30 -0
- memdiver/core/binary_formats/navigator.py +227 -0
- memdiver/core/binary_formats/pe_defs.py +42 -0
- memdiver/core/classify.py +40 -0
- memdiver/core/config_schema.py +113 -0
- memdiver/core/constants.py +39 -0
- memdiver/core/dataset_metadata.py +186 -0
- memdiver/core/discovery.py +468 -0
- memdiver/core/display_labels.py +23 -0
- memdiver/core/dump_driver.py +204 -0
- memdiver/core/dump_io.py +108 -0
- memdiver/core/dump_search.py +70 -0
- memdiver/core/dump_source.py +364 -0
- memdiver/core/entropy.py +168 -0
- memdiver/core/format_detect.py +136 -0
- memdiver/core/input_schemas.py +62 -0
- memdiver/core/kdf.py +196 -0
- memdiver/core/kdf_base.py +74 -0
- memdiver/core/kdf_registry.py +80 -0
- memdiver/core/kdf_ssh.py +176 -0
- memdiver/core/kdf_tls.py +181 -0
- memdiver/core/keylog.py +87 -0
- memdiver/core/keylog_templates.py +108 -0
- memdiver/core/log.py +49 -0
- memdiver/core/models.py +153 -0
- memdiver/core/msl_helpers.py +26 -0
- memdiver/core/phase_normalizer.py +147 -0
- memdiver/core/proc_maps_parser.py +113 -0
- memdiver/core/protocols.py +150 -0
- memdiver/core/region_align.py +218 -0
- memdiver/core/region_analysis.py +122 -0
- memdiver/core/strings.py +143 -0
- memdiver/core/structure_defs.py +157 -0
- memdiver/core/structure_library.py +105 -0
- memdiver/core/structure_library_ssh.py +42 -0
- memdiver/core/structure_library_tls.py +169 -0
- memdiver/core/structure_loader.py +43 -0
- memdiver/core/structure_overlay.py +241 -0
- memdiver/core/structure_schema.py +119 -0
- memdiver/core/variance.py +195 -0
- memdiver/engine/__init__.py +0 -0
- memdiver/engine/batch.py +196 -0
- memdiver/engine/batch_task_runner.py +210 -0
- memdiver/engine/brute_force.py +446 -0
- memdiver/engine/cancellation.py +36 -0
- memdiver/engine/candidate_pipeline.py +337 -0
- memdiver/engine/consensus.py +329 -0
- memdiver/engine/consensus_msl.py +182 -0
- memdiver/engine/convergence.py +181 -0
- memdiver/engine/correlator.py +179 -0
- memdiver/engine/derived_keys.py +70 -0
- memdiver/engine/diff_store.py +195 -0
- memdiver/engine/experiment_task_runner.py +458 -0
- memdiver/engine/nsweep.py +324 -0
- memdiver/engine/oracle.py +145 -0
- memdiver/engine/pipeline.py +250 -0
- memdiver/engine/pipeline_runner.py +586 -0
- memdiver/engine/progress.py +104 -0
- memdiver/engine/project_db.py +193 -0
- memdiver/engine/results.py +79 -0
- memdiver/engine/serializer.py +165 -0
- memdiver/engine/session_store.py +234 -0
- memdiver/engine/verification.py +131 -0
- memdiver/engine/vol3_emit.py +245 -0
- memdiver/engine/worker.py +57 -0
- memdiver/frontend/dist/assets/EntropyChart-8M8HrU-B.js +1 -0
- memdiver/frontend/dist/assets/EntropyChart-DAtoAJCY.js +1 -0
- memdiver/frontend/dist/assets/SurvivorCurve-BY20wrM1.js +1 -0
- memdiver/frontend/dist/assets/SurvivorCurve-D9JxV81S.js +1 -0
- memdiver/frontend/dist/assets/analysis-store-BpQfEZUW.js +1 -0
- memdiver/frontend/dist/assets/client-CNl5BiOS.js +1 -0
- memdiver/frontend/dist/assets/dump-store-Dwu_Tzhp.js +1 -0
- memdiver/frontend/dist/assets/hex-store-CpkAyGnC.js +1 -0
- memdiver/frontend/dist/assets/hex-store-gtC9qvXO.js +1 -0
- memdiver/frontend/dist/assets/index-CkDrSf6P.js +87 -0
- memdiver/frontend/dist/assets/index-PoxQeNeU.css +2 -0
- memdiver/frontend/dist/assets/jsx-runtime-sPcSWLTJ.js +1 -0
- memdiver/frontend/dist/assets/normalize-plot-twAkls4V.js +4030 -0
- memdiver/frontend/dist/assets/primitives-CAn_uvh6.js +1 -0
- memdiver/frontend/dist/assets/react-BcFANYnC.js +1 -0
- memdiver/frontend/dist/assets/react-CYzKIDNi.js +1 -0
- memdiver/frontend/dist/assets/results-store-BSf3LrQV.js +1 -0
- memdiver/frontend/dist/assets/strings-store-CicX0gXn.js +1 -0
- memdiver/frontend/dist/assets/survivor-traces-DyBxrQCs.js +1 -0
- memdiver/frontend/dist/assets/useChartTheme-7Z3828aq.js +1 -0
- memdiver/frontend/dist/favicon.svg +1 -0
- memdiver/frontend/dist/icons.svg +24 -0
- memdiver/frontend/dist/index.html +30 -0
- memdiver/frontend/dist/memdiver-logo.svg +352 -0
- memdiver/harvester/__init__.py +0 -0
- memdiver/harvester/ingestor.py +90 -0
- memdiver/harvester/metadata_store.py +83 -0
- memdiver/harvester/sidecar.py +78 -0
- memdiver/legacy_app.py +145 -0
- memdiver/mcp_server/__init__.py +12 -0
- memdiver/mcp_server/server.py +208 -0
- memdiver/mcp_server/session.py +67 -0
- memdiver/mcp_server/tools.py +125 -0
- memdiver/mcp_server/tools_inspect.py +352 -0
- memdiver/mcp_server/tools_pipeline.py +226 -0
- memdiver/mcp_server/tools_xref.py +86 -0
- memdiver/msl/__init__.py +53 -0
- memdiver/msl/block_iter.py +57 -0
- memdiver/msl/block_tree.py +81 -0
- memdiver/msl/compress.py +73 -0
- memdiver/msl/decoders.py +590 -0
- memdiver/msl/decoders_ext.py +287 -0
- memdiver/msl/enums.py +184 -0
- memdiver/msl/hashing.py +45 -0
- memdiver/msl/importer.py +122 -0
- memdiver/msl/integrity.py +72 -0
- memdiver/msl/key_extract.py +168 -0
- memdiver/msl/page_map.py +135 -0
- memdiver/msl/reader.py +256 -0
- memdiver/msl/session_extract.py +156 -0
- memdiver/msl/string_extract.py +139 -0
- memdiver/msl/types.py +387 -0
- memdiver/msl/writer.py +217 -0
- memdiver/msl/xref_resolver.py +125 -0
- memdiver/run.py +874 -0
- memdiver/ui/__init__.py +0 -0
- memdiver/ui/adapters.py +56 -0
- memdiver/ui/components/__init__.py +0 -0
- memdiver/ui/components/bookmark_store.py +70 -0
- memdiver/ui/components/color_scheme.py +70 -0
- memdiver/ui/components/header.py +67 -0
- memdiver/ui/components/hex_pager.py +52 -0
- memdiver/ui/components/hex_renderer.py +117 -0
- memdiver/ui/components/html_builder.py +89 -0
- memdiver/ui/components/progress_helper.py +54 -0
- memdiver/ui/components/setup_wizard.py +158 -0
- memdiver/ui/controls/__init__.py +0 -0
- memdiver/ui/controls/analysis_panel.py +139 -0
- memdiver/ui/controls/config_panel.py +101 -0
- memdiver/ui/controls/selector_panel.py +169 -0
- memdiver/ui/locales.py +37 -0
- memdiver/ui/mode.py +107 -0
- memdiver/ui/nicegui/__init__.py +1 -0
- memdiver/ui/nicegui/api_routes.py +74 -0
- memdiver/ui/nicegui/theme.py +74 -0
- memdiver/ui/nicegui/wizard.py +104 -0
- memdiver/ui/nicegui/workspace.py +84 -0
- memdiver/ui/state.py +137 -0
- memdiver/ui/views/__init__.py +0 -0
- memdiver/ui/views/block_navigator.py +288 -0
- memdiver/ui/views/consensus_view.py +71 -0
- memdiver/ui/views/cross_library.py +69 -0
- memdiver/ui/views/differential_diff.py +90 -0
- memdiver/ui/views/entropy_chart.py +82 -0
- memdiver/ui/views/heatmap.py +97 -0
- memdiver/ui/views/hex_comparison.py +116 -0
- memdiver/ui/views/hex_navigator.py +171 -0
- memdiver/ui/views/hex_viewer.py +125 -0
- memdiver/ui/views/investigation_panel.py +122 -0
- memdiver/ui/views/phase_lifecycle.py +76 -0
- memdiver/ui/views/session_view.py +135 -0
- memdiver/ui/views/variance_map.py +80 -0
- memdiver/ui/views/vas_view.py +185 -0
- memdiver-0.5.6.dist-info/METADATA +234 -0
- memdiver-0.5.6.dist-info/RECORD +232 -0
- memdiver-0.5.6.dist-info/WHEEL +5 -0
- memdiver-0.5.6.dist-info/entry_points.txt +2 -0
- memdiver-0.5.6.dist-info/licenses/LICENSE +201 -0
- memdiver-0.5.6.dist-info/top_level.txt +1 -0
memdiver/__init__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""MemDiver - Interactive platform for identifying and analyzing data structures in memory dumps."""
|
|
2
|
+
|
|
3
|
+
from importlib.metadata import PackageNotFoundError, version as _version
|
|
4
|
+
|
|
5
|
+
try:
|
|
6
|
+
__version__ = _version("memdiver")
|
|
7
|
+
except PackageNotFoundError:
|
|
8
|
+
__version__ = "0.0.0+unknown"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Algorithm plugin system for memory dump analysis."""
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Base classes for analysis algorithms."""
|
|
2
|
+
|
|
3
|
+
from abc import ABC, abstractmethod
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from typing import Any, Dict, List, Optional
|
|
6
|
+
|
|
7
|
+
from core.models import CryptoSecret, deprecated_kwarg
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass
|
|
11
|
+
class AnalysisContext:
|
|
12
|
+
"""Context provided to algorithms."""
|
|
13
|
+
library: str
|
|
14
|
+
protocol_version: str
|
|
15
|
+
phase: str
|
|
16
|
+
secrets: List[CryptoSecret] = field(default_factory=list)
|
|
17
|
+
extra: Dict[str, Any] = field(default_factory=dict)
|
|
18
|
+
|
|
19
|
+
@property
|
|
20
|
+
def tls_version(self) -> str:
|
|
21
|
+
"""Backward-compatible alias for protocol_version."""
|
|
22
|
+
return self.protocol_version
|
|
23
|
+
|
|
24
|
+
@tls_version.setter
|
|
25
|
+
def tls_version(self, value: str) -> None:
|
|
26
|
+
self.protocol_version = value
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
deprecated_kwarg(AnalysisContext, "tls_version", "protocol_version")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class Match:
|
|
34
|
+
"""A single match found by an algorithm."""
|
|
35
|
+
offset: int
|
|
36
|
+
length: int
|
|
37
|
+
confidence: float
|
|
38
|
+
label: str = ""
|
|
39
|
+
data: bytes = b""
|
|
40
|
+
metadata: Dict[str, Any] = field(default_factory=dict)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass
|
|
44
|
+
class AlgorithmResult:
|
|
45
|
+
"""Result from running an algorithm."""
|
|
46
|
+
algorithm_name: str
|
|
47
|
+
confidence: float
|
|
48
|
+
matches: List[Match] = field(default_factory=list)
|
|
49
|
+
metadata: Dict[str, Any] = field(default_factory=dict)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class BaseAlgorithm(ABC):
|
|
53
|
+
"""Base class for all analysis algorithms."""
|
|
54
|
+
|
|
55
|
+
name: str = ""
|
|
56
|
+
description: str = ""
|
|
57
|
+
mode: str = "" # Use KNOWN_KEY or UNKNOWN_KEY from core.constants
|
|
58
|
+
|
|
59
|
+
@abstractmethod
|
|
60
|
+
def run(self, dump_data: bytes, context: AnalysisContext) -> AlgorithmResult:
|
|
61
|
+
"""Run the algorithm on dump data."""
|
|
62
|
+
...
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Known-key algorithms - require ground truth secrets."""
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Exact byte match algorithm - search for known secret bytes in dump."""
|
|
2
|
+
|
|
3
|
+
from algorithms.base import AlgorithmResult, AnalysisContext, BaseAlgorithm, Match
|
|
4
|
+
from core.constants import KNOWN_KEY
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ExactMatchAlgorithm(BaseAlgorithm):
|
|
8
|
+
"""Search for exact secret byte sequences in memory dumps."""
|
|
9
|
+
|
|
10
|
+
name = "exact_match"
|
|
11
|
+
description = "Search for exact secret byte sequences"
|
|
12
|
+
mode = KNOWN_KEY
|
|
13
|
+
|
|
14
|
+
def run(self, dump_data: bytes, context: AnalysisContext) -> AlgorithmResult:
|
|
15
|
+
matches = []
|
|
16
|
+
|
|
17
|
+
for secret in context.secrets:
|
|
18
|
+
needle = secret.secret_value
|
|
19
|
+
start = 0
|
|
20
|
+
while True:
|
|
21
|
+
idx = dump_data.find(needle, start)
|
|
22
|
+
if idx == -1:
|
|
23
|
+
break
|
|
24
|
+
matches.append(Match(
|
|
25
|
+
offset=idx,
|
|
26
|
+
length=len(needle),
|
|
27
|
+
confidence=1.0,
|
|
28
|
+
label=secret.secret_type,
|
|
29
|
+
data=needle,
|
|
30
|
+
metadata={"secret_type": secret.secret_type},
|
|
31
|
+
))
|
|
32
|
+
start = idx + 1
|
|
33
|
+
|
|
34
|
+
found_types = set(m.label for m in matches)
|
|
35
|
+
total_types = len(set(s.secret_type for s in context.secrets))
|
|
36
|
+
confidence = len(found_types) / total_types if total_types > 0 else 0.0
|
|
37
|
+
|
|
38
|
+
return AlgorithmResult(
|
|
39
|
+
algorithm_name=self.name,
|
|
40
|
+
confidence=confidence,
|
|
41
|
+
matches=matches,
|
|
42
|
+
metadata={
|
|
43
|
+
"total_secret_types": total_types,
|
|
44
|
+
"found_secret_types": len(found_types),
|
|
45
|
+
"found_types_list": sorted(found_types),
|
|
46
|
+
},
|
|
47
|
+
)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Pattern-based algorithms for structural matching."""
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "boringssl_tls13_exporter_secret",
|
|
3
|
+
"description": "Structural pattern for BoringSSL TLS 1.3 EXPORTER_SECRET in memory",
|
|
4
|
+
"applicable_to": {
|
|
5
|
+
"libraries": ["boringssl"],
|
|
6
|
+
"protocol_versions": ["13"]
|
|
7
|
+
},
|
|
8
|
+
"key_spec": {
|
|
9
|
+
"length": 32,
|
|
10
|
+
"entropy_min": 7.0
|
|
11
|
+
},
|
|
12
|
+
"pattern": {
|
|
13
|
+
"before": [
|
|
14
|
+
{
|
|
15
|
+
"offset": -8,
|
|
16
|
+
"bytes": "00 00 00 00 00 00 00 20",
|
|
17
|
+
"mask": "00 00 00 00 00 00 00 ff"
|
|
18
|
+
}
|
|
19
|
+
],
|
|
20
|
+
"after": [
|
|
21
|
+
{
|
|
22
|
+
"offset": 32,
|
|
23
|
+
"bytes": "00 00 00 00 00 00 00 00",
|
|
24
|
+
"mask": "00 00 00 00 00 00 00 00"
|
|
25
|
+
}
|
|
26
|
+
]
|
|
27
|
+
}
|
|
28
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "openssh_ssh2_session_key",
|
|
3
|
+
"description": "Structural pattern for OpenSSH SSH-2 session key in memory",
|
|
4
|
+
"applicable_to": {
|
|
5
|
+
"libraries": ["openssh"],
|
|
6
|
+
"protocol_versions": ["2"]
|
|
7
|
+
},
|
|
8
|
+
"key_spec": {
|
|
9
|
+
"length": 32,
|
|
10
|
+
"entropy_min": 7.0
|
|
11
|
+
},
|
|
12
|
+
"pattern": {
|
|
13
|
+
"before": [
|
|
14
|
+
{
|
|
15
|
+
"offset": -4,
|
|
16
|
+
"bytes": "00 00 00 20",
|
|
17
|
+
"mask": "00 00 00 ff"
|
|
18
|
+
}
|
|
19
|
+
],
|
|
20
|
+
"after": [
|
|
21
|
+
{
|
|
22
|
+
"offset": 32,
|
|
23
|
+
"bytes": "00 00 00 00",
|
|
24
|
+
"mask": "00 00 00 00"
|
|
25
|
+
}
|
|
26
|
+
]
|
|
27
|
+
}
|
|
28
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "openssl_tls12_master_secret",
|
|
3
|
+
"description": "Structural pattern for OpenSSL TLS 1.2 master secret in memory",
|
|
4
|
+
"applicable_to": {
|
|
5
|
+
"libraries": ["openssl"],
|
|
6
|
+
"protocol_versions": ["12"]
|
|
7
|
+
},
|
|
8
|
+
"key_spec": {
|
|
9
|
+
"length": 48,
|
|
10
|
+
"entropy_min": 7.0
|
|
11
|
+
},
|
|
12
|
+
"pattern": {
|
|
13
|
+
"before": [
|
|
14
|
+
{
|
|
15
|
+
"offset": -4,
|
|
16
|
+
"bytes": "00 00 00 30",
|
|
17
|
+
"mask": "00 00 00 ff"
|
|
18
|
+
}
|
|
19
|
+
],
|
|
20
|
+
"after": [
|
|
21
|
+
{
|
|
22
|
+
"offset": 48,
|
|
23
|
+
"bytes": "00 00 00 00 00 00 00 00",
|
|
24
|
+
"mask": "00 00 00 00 00 00 00 00"
|
|
25
|
+
}
|
|
26
|
+
]
|
|
27
|
+
}
|
|
28
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""Auto-discovery registry for algorithm plugins."""
|
|
2
|
+
|
|
3
|
+
import importlib
|
|
4
|
+
import pkgutil
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Dict, List
|
|
7
|
+
|
|
8
|
+
from core.constants import AlgorithmMode
|
|
9
|
+
|
|
10
|
+
from .base import BaseAlgorithm
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class AlgorithmRegistry:
|
|
14
|
+
"""Discover and manage algorithm plugins."""
|
|
15
|
+
|
|
16
|
+
def __init__(self):
|
|
17
|
+
self._algorithms: Dict[str, BaseAlgorithm] = {}
|
|
18
|
+
|
|
19
|
+
def discover(self) -> None:
|
|
20
|
+
"""Walk known_key/ and unknown_key/ subdirectories to find algorithms."""
|
|
21
|
+
base_dir = Path(__file__).parent
|
|
22
|
+
|
|
23
|
+
for subdir in ["known_key", "unknown_key", "patterns"]:
|
|
24
|
+
pkg_path = base_dir / subdir
|
|
25
|
+
if not pkg_path.is_dir():
|
|
26
|
+
continue
|
|
27
|
+
|
|
28
|
+
pkg_name = f"algorithms.{subdir}"
|
|
29
|
+
try:
|
|
30
|
+
pkg = importlib.import_module(pkg_name)
|
|
31
|
+
except ImportError:
|
|
32
|
+
continue
|
|
33
|
+
|
|
34
|
+
for importer, modname, ispkg in pkgutil.walk_packages(
|
|
35
|
+
path=[str(pkg_path)], prefix=f"{pkg_name}."
|
|
36
|
+
):
|
|
37
|
+
try:
|
|
38
|
+
mod = importlib.import_module(modname)
|
|
39
|
+
except ImportError:
|
|
40
|
+
continue
|
|
41
|
+
|
|
42
|
+
for attr_name in dir(mod):
|
|
43
|
+
attr = getattr(mod, attr_name)
|
|
44
|
+
if (isinstance(attr, type)
|
|
45
|
+
and issubclass(attr, BaseAlgorithm)
|
|
46
|
+
and attr is not BaseAlgorithm
|
|
47
|
+
and hasattr(attr, 'name')
|
|
48
|
+
and attr.name):
|
|
49
|
+
instance = attr()
|
|
50
|
+
self._algorithms[instance.name] = instance
|
|
51
|
+
|
|
52
|
+
def get(self, name: str) -> BaseAlgorithm:
|
|
53
|
+
return self._algorithms[name]
|
|
54
|
+
|
|
55
|
+
def list_all(self) -> List[BaseAlgorithm]:
|
|
56
|
+
return list(self._algorithms.values())
|
|
57
|
+
|
|
58
|
+
def list_by_mode(self, mode: AlgorithmMode) -> List[BaseAlgorithm]:
|
|
59
|
+
return [a for a in self._algorithms.values() if a.mode == mode]
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def names(self) -> List[str]:
|
|
63
|
+
return sorted(self._algorithms.keys())
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
_registry = None
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def get_registry() -> AlgorithmRegistry:
|
|
70
|
+
global _registry
|
|
71
|
+
if _registry is None:
|
|
72
|
+
_registry = AlgorithmRegistry()
|
|
73
|
+
_registry.discover()
|
|
74
|
+
return _registry
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Unknown-key algorithms - detect keys without ground truth."""
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
"""Entropy change-point detection using CUSUM for finding key-material plateaus."""
|
|
2
|
+
|
|
3
|
+
from bisect import bisect_left, bisect_right
|
|
4
|
+
from typing import Dict, List, Tuple
|
|
5
|
+
|
|
6
|
+
from algorithms.base import AlgorithmResult, AnalysisContext, BaseAlgorithm, Match
|
|
7
|
+
from core.constants import UNKNOWN_KEY
|
|
8
|
+
from core.entropy import compute_entropy_profile
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class ChangePointAlgorithm(BaseAlgorithm):
|
|
12
|
+
"""Detect key-material plateaus via CUSUM entropy change-point analysis.
|
|
13
|
+
|
|
14
|
+
Three stages:
|
|
15
|
+
1. Entropy profiling via sliding-window Shannon entropy.
|
|
16
|
+
2. Gradient CUSUM edge detection on the first difference of the profile.
|
|
17
|
+
3. Plateau extraction by pairing rising/falling edges that match
|
|
18
|
+
expected key widths and exceed the entropy threshold.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
name = "change_point"
|
|
22
|
+
description = "Entropy change-point detection using CUSUM to find key-material plateaus"
|
|
23
|
+
mode = UNKNOWN_KEY
|
|
24
|
+
|
|
25
|
+
DEFAULT_WINDOW = 32
|
|
26
|
+
DEFAULT_ENTROPY_THRESHOLD = 4.5
|
|
27
|
+
DEFAULT_CUSUM_THRESHOLD = 0.8
|
|
28
|
+
DEFAULT_DRIFT = 0.05
|
|
29
|
+
DEFAULT_STEP = 16
|
|
30
|
+
DEFAULT_PLATEAU_WIDTHS = [32, 48]
|
|
31
|
+
|
|
32
|
+
@staticmethod
|
|
33
|
+
def _cusum_change_points(
|
|
34
|
+
profile: List[Tuple[int, float]],
|
|
35
|
+
threshold: float = 0.8,
|
|
36
|
+
drift: float = 0.05,
|
|
37
|
+
) -> List[Tuple[int, str, float]]:
|
|
38
|
+
"""Detect change points using gradient CUSUM.
|
|
39
|
+
|
|
40
|
+
Reports the onset offset (not detection offset) for each edge.
|
|
41
|
+
Returns list of (offset, direction, cusum_value) tuples where
|
|
42
|
+
direction is "up" (rising) or "down" (falling).
|
|
43
|
+
"""
|
|
44
|
+
if len(profile) < 2:
|
|
45
|
+
return []
|
|
46
|
+
|
|
47
|
+
s_high = 0.0
|
|
48
|
+
s_low = 0.0
|
|
49
|
+
change_points: List[Tuple[int, str, float]] = []
|
|
50
|
+
|
|
51
|
+
# Track where the current accumulation run started so we can
|
|
52
|
+
# report the onset offset rather than the detection offset.
|
|
53
|
+
run_start_high = profile[0][0]
|
|
54
|
+
run_start_low = profile[0][0]
|
|
55
|
+
|
|
56
|
+
for i in range(1, len(profile)):
|
|
57
|
+
offset = profile[i][0]
|
|
58
|
+
gradient = profile[i][1] - profile[i - 1][1]
|
|
59
|
+
|
|
60
|
+
# Track run-start: reset when the statistic was at zero.
|
|
61
|
+
prev_s_high = s_high
|
|
62
|
+
prev_s_low = s_low
|
|
63
|
+
|
|
64
|
+
s_high = max(0.0, s_high + gradient - drift)
|
|
65
|
+
s_low = max(0.0, s_low - gradient - drift)
|
|
66
|
+
|
|
67
|
+
if prev_s_high == 0.0 and s_high > 0.0:
|
|
68
|
+
run_start_high = offset
|
|
69
|
+
if prev_s_low == 0.0 and s_low > 0.0:
|
|
70
|
+
run_start_low = offset
|
|
71
|
+
|
|
72
|
+
if s_high > threshold:
|
|
73
|
+
change_points.append((run_start_high, "up", s_high))
|
|
74
|
+
s_high = 0.0
|
|
75
|
+
|
|
76
|
+
if s_low > threshold:
|
|
77
|
+
change_points.append((run_start_low, "down", s_low))
|
|
78
|
+
s_low = 0.0
|
|
79
|
+
|
|
80
|
+
return change_points
|
|
81
|
+
|
|
82
|
+
def _extract_plateaus(
|
|
83
|
+
self,
|
|
84
|
+
dump_data: bytes,
|
|
85
|
+
profile: List[Tuple[int, float]],
|
|
86
|
+
change_points: List[Tuple[int, str, float]],
|
|
87
|
+
entropy_threshold: float,
|
|
88
|
+
plateau_widths: List[int],
|
|
89
|
+
window: int,
|
|
90
|
+
) -> List[Match]:
|
|
91
|
+
"""Pair rising/falling edges into key-sized high-entropy plateaus.
|
|
92
|
+
|
|
93
|
+
Accepts pairs whose span matches a target key width (adjusted for
|
|
94
|
+
window smearing) and whose mean entropy exceeds the threshold.
|
|
95
|
+
"""
|
|
96
|
+
# Build an offset -> entropy lookup for fast slicing.
|
|
97
|
+
entropy_lookup: Dict[int, float] = {off: ent for off, ent in profile}
|
|
98
|
+
sorted_offsets = sorted(entropy_lookup.keys())
|
|
99
|
+
|
|
100
|
+
# Pair consecutive up -> down transitions.
|
|
101
|
+
pairs: List[Tuple[int, int, float, float]] = []
|
|
102
|
+
i = 0
|
|
103
|
+
while i < len(change_points) - 1:
|
|
104
|
+
offset_up, direction_up, strength_up = change_points[i]
|
|
105
|
+
offset_down, direction_down, strength_down = change_points[i + 1]
|
|
106
|
+
|
|
107
|
+
if direction_up == "up" and direction_down == "down":
|
|
108
|
+
pairs.append((offset_up, offset_down, strength_up, strength_down))
|
|
109
|
+
i += 2
|
|
110
|
+
else:
|
|
111
|
+
i += 1
|
|
112
|
+
|
|
113
|
+
width_tolerance = 16
|
|
114
|
+
matches: List[Match] = []
|
|
115
|
+
|
|
116
|
+
for start_off, end_off, strength_up, strength_down in pairs:
|
|
117
|
+
profile_span = end_off - start_off
|
|
118
|
+
if profile_span <= 0:
|
|
119
|
+
continue
|
|
120
|
+
|
|
121
|
+
best_key_width = 0
|
|
122
|
+
best_distance = width_tolerance + 1
|
|
123
|
+
for target_w in plateau_widths:
|
|
124
|
+
expected_spans = [target_w, target_w + window - 1]
|
|
125
|
+
for expected in expected_spans:
|
|
126
|
+
dist = abs(profile_span - expected)
|
|
127
|
+
if dist <= width_tolerance and dist < best_distance:
|
|
128
|
+
best_distance = dist
|
|
129
|
+
best_key_width = target_w
|
|
130
|
+
|
|
131
|
+
if best_key_width == 0:
|
|
132
|
+
continue
|
|
133
|
+
|
|
134
|
+
# Compute mean entropy over the plateau region (binary search).
|
|
135
|
+
lo = bisect_left(sorted_offsets, start_off)
|
|
136
|
+
hi = bisect_right(sorted_offsets, end_off)
|
|
137
|
+
segment_offsets = sorted_offsets[lo:hi]
|
|
138
|
+
if not segment_offsets:
|
|
139
|
+
continue
|
|
140
|
+
|
|
141
|
+
segment_entropies = [entropy_lookup[off] for off in segment_offsets]
|
|
142
|
+
mean_entropy = sum(segment_entropies) / len(segment_entropies)
|
|
143
|
+
|
|
144
|
+
if mean_entropy < entropy_threshold:
|
|
145
|
+
continue
|
|
146
|
+
|
|
147
|
+
# The key material starts at the rising edge. Extract
|
|
148
|
+
# best_key_width bytes from that point.
|
|
149
|
+
key_start = start_off
|
|
150
|
+
key_len = min(best_key_width, len(dump_data) - key_start)
|
|
151
|
+
plateau_data = dump_data[key_start:key_start + key_len]
|
|
152
|
+
|
|
153
|
+
cusum_strength = (strength_up + strength_down) / 2.0
|
|
154
|
+
confidence = min(mean_entropy / 8.0, 1.0) * min(cusum_strength / 3.0, 1.0)
|
|
155
|
+
|
|
156
|
+
matches.append(
|
|
157
|
+
Match(
|
|
158
|
+
offset=key_start,
|
|
159
|
+
length=key_len,
|
|
160
|
+
confidence=round(confidence, 4),
|
|
161
|
+
label=f"cusum_plateau_{key_len}B",
|
|
162
|
+
data=plateau_data,
|
|
163
|
+
metadata={
|
|
164
|
+
"entropy_mean": round(mean_entropy, 4),
|
|
165
|
+
"plateau_width": profile_span,
|
|
166
|
+
"estimated_key_width": best_key_width,
|
|
167
|
+
"cusum_strength": round(cusum_strength, 4),
|
|
168
|
+
"profile_segment": [
|
|
169
|
+
round(e, 4) for e in segment_entropies[:10]
|
|
170
|
+
],
|
|
171
|
+
},
|
|
172
|
+
)
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
return matches
|
|
176
|
+
|
|
177
|
+
def run(self, dump_data: bytes, context: AnalysisContext) -> AlgorithmResult:
|
|
178
|
+
"""Run CUSUM change-point detection. Parameters from context.extra."""
|
|
179
|
+
window = context.extra.get("window", self.DEFAULT_WINDOW)
|
|
180
|
+
step = context.extra.get("step", self.DEFAULT_STEP)
|
|
181
|
+
entropy_threshold = context.extra.get(
|
|
182
|
+
"entropy_threshold", self.DEFAULT_ENTROPY_THRESHOLD
|
|
183
|
+
)
|
|
184
|
+
cusum_threshold = context.extra.get(
|
|
185
|
+
"cusum_threshold", self.DEFAULT_CUSUM_THRESHOLD
|
|
186
|
+
)
|
|
187
|
+
drift = context.extra.get("drift", self.DEFAULT_DRIFT)
|
|
188
|
+
plateau_widths = context.extra.get(
|
|
189
|
+
"plateau_widths", self.DEFAULT_PLATEAU_WIDTHS
|
|
190
|
+
)
|
|
191
|
+
|
|
192
|
+
# Stage 1: compute entropy profile.
|
|
193
|
+
profile = compute_entropy_profile(dump_data, window=window, step=step)
|
|
194
|
+
|
|
195
|
+
# Stage 2: detect change points via gradient CUSUM.
|
|
196
|
+
change_points = self._cusum_change_points(
|
|
197
|
+
profile, threshold=cusum_threshold, drift=drift
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
# Stage 3: extract qualifying plateaus.
|
|
201
|
+
matches = self._extract_plateaus(
|
|
202
|
+
dump_data, profile, change_points, entropy_threshold, plateau_widths,
|
|
203
|
+
window,
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
# Overall confidence scales with the number of plausible matches.
|
|
207
|
+
overall_confidence = min(len(matches) / 10.0, 1.0) if matches else 0.0
|
|
208
|
+
|
|
209
|
+
return AlgorithmResult(
|
|
210
|
+
algorithm_name=self.name,
|
|
211
|
+
confidence=round(overall_confidence, 4),
|
|
212
|
+
matches=matches,
|
|
213
|
+
metadata={
|
|
214
|
+
"window": window,
|
|
215
|
+
"step": step,
|
|
216
|
+
"entropy_threshold": entropy_threshold,
|
|
217
|
+
"cusum_threshold": cusum_threshold,
|
|
218
|
+
"drift": drift,
|
|
219
|
+
"plateau_widths": plateau_widths,
|
|
220
|
+
"change_points_detected": len(change_points),
|
|
221
|
+
"plateaus_matched": len(matches),
|
|
222
|
+
},
|
|
223
|
+
)
|