izayoi 0.5.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- izayoi/__init__.py +29 -0
- izayoi/cli.py +227 -0
- izayoi/diffruns.py +143 -0
- izayoi/entropy.py +59 -0
- izayoi/fastpath.py +269 -0
- izayoi/images.py +36 -0
- izayoi/pairs.py +216 -0
- izayoi/report.py +195 -0
- izayoi/triage.py +287 -0
- izayoi-0.5.2.dist-info/METADATA +416 -0
- izayoi-0.5.2.dist-info/RECORD +15 -0
- izayoi-0.5.2.dist-info/WHEEL +5 -0
- izayoi-0.5.2.dist-info/entry_points.txt +2 -0
- izayoi-0.5.2.dist-info/licenses/LICENSE +201 -0
- izayoi-0.5.2.dist-info/top_level.txt +1 -0
izayoi/__init__.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""izayoi — 複数時点メモリダンプ比較ツールキット。
|
|
2
|
+
|
|
3
|
+
ページ単位のエントロピー分析、ダンプ間の差分ラン列挙、
|
|
4
|
+
複数時点ダンプのページ分類(static/flip/settled/volatile)、
|
|
5
|
+
暗文↔平文ページ対の推定、ダンプのトリアージ
|
|
6
|
+
(PEヘッダ検出・読みやすい資産の所在推定)を提供する。
|
|
7
|
+
標準ライブラリのみで動作し、NumPy が導入されていれば
|
|
8
|
+
ホットパスが自動的にベクトル化実装に切り替わる(fastpath)。
|
|
9
|
+
"""
|
|
10
|
+
from .diffruns import classify_pages, diff_runs, label_ranges, summarize
|
|
11
|
+
from .entropy import entropy_map, high_entropy_ranges, shannon_entropy
|
|
12
|
+
from .images import fingerprint, load_image, strip_header
|
|
13
|
+
from .pairs import (analyze_pairs, code_density, descriptor_suggestion,
|
|
14
|
+
find_pairs, page_class, printable_ratio,
|
|
15
|
+
refine_with_diffs, zero_ratio)
|
|
16
|
+
from .triage import (detect_pe, page_census, readability_ranking,
|
|
17
|
+
readable_sources, region_info, triage, triage_images)
|
|
18
|
+
|
|
19
|
+
__version__ = "0.5.2"
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"analyze_pairs", "classify_pages", "code_density", "detect_pe",
|
|
23
|
+
"descriptor_suggestion", "diff_runs", "entropy_map", "fingerprint",
|
|
24
|
+
"find_pairs", "high_entropy_ranges", "label_ranges", "load_image",
|
|
25
|
+
"page_class", "page_census", "printable_ratio", "readability_ranking",
|
|
26
|
+
"readable_sources", "refine_with_diffs", "region_info", "shannon_entropy",
|
|
27
|
+
"strip_header", "summarize", "triage", "triage_images", "zero_ratio",
|
|
28
|
+
"__version__",
|
|
29
|
+
]
|
izayoi/cli.py
ADDED
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
"""izayoi コマンドライン。
|
|
2
|
+
|
|
3
|
+
使い方: python -m izayoi.cli <cmd> ...
|
|
4
|
+
"""
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import argparse
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import sys
|
|
11
|
+
|
|
12
|
+
from . import __version__
|
|
13
|
+
|
|
14
|
+
from .diffruns import classify_pages, diff_runs, label_ranges, summarize
|
|
15
|
+
from .entropy import high_entropy_ranges
|
|
16
|
+
from .images import load_image
|
|
17
|
+
from .pairs import analyze_pairs
|
|
18
|
+
from .report import json_report, markdown_report
|
|
19
|
+
from .triage import triage
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _int(s: str) -> int:
|
|
23
|
+
return int(s, 0)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _cmd_entropy(args) -> int:
|
|
27
|
+
data = load_image(args.image)
|
|
28
|
+
ranges = high_entropy_ranges(data, args.threshold, args.page)
|
|
29
|
+
if not ranges:
|
|
30
|
+
print("高エントロピー領域なし (threshold=%.2f)" % args.threshold)
|
|
31
|
+
return 0
|
|
32
|
+
print("%d 領域 (threshold=%.2f, page=0x%x):"
|
|
33
|
+
% (len(ranges), args.threshold, args.page))
|
|
34
|
+
for start, end, mx in ranges:
|
|
35
|
+
print(" 0x%08x-0x%08x %6d bytes max=%.2f"
|
|
36
|
+
% (start, end, end - start, mx))
|
|
37
|
+
return 0
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _cmd_diff(args) -> int:
|
|
41
|
+
a, b = load_image(args.a), load_image(args.b)
|
|
42
|
+
runs = diff_runs(a, b, merge_gap=args.merge_gap, min_length=args.min_length)
|
|
43
|
+
if not runs:
|
|
44
|
+
print("差分なし")
|
|
45
|
+
return 0
|
|
46
|
+
total = sum(r["length"] for r in runs)
|
|
47
|
+
print("%d ラン / %d bytes (merge_gap=%d, min_length=%d):"
|
|
48
|
+
% (len(runs), total, args.merge_gap, args.min_length))
|
|
49
|
+
for r in runs[:args.limit]:
|
|
50
|
+
print(" 0x%08x %8d %s" % (r["offset"], r["length"], r["kind"]))
|
|
51
|
+
if len(runs) > args.limit:
|
|
52
|
+
print(" ... ほか %d ラン" % (len(runs) - args.limit))
|
|
53
|
+
return 0
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _cmd_compare(args) -> int:
|
|
57
|
+
images = [load_image(p) for p in args.images]
|
|
58
|
+
items = classify_pages(images, args.page)
|
|
59
|
+
acc = summarize(items, args.page)
|
|
60
|
+
print("ページ分類 (page=0x%x):" % args.page)
|
|
61
|
+
for label, (npages, nbytes) in acc.items():
|
|
62
|
+
print(" %-8s %5d ページ / %8d bytes" % (label, npages, nbytes))
|
|
63
|
+
if args.ranges:
|
|
64
|
+
print("領域一覧:")
|
|
65
|
+
for label, ranges in label_ranges(items, args.page).items():
|
|
66
|
+
for s, e in ranges:
|
|
67
|
+
print(" %-8s 0x%08x-0x%08x (%d bytes)"
|
|
68
|
+
% (label, s, e, e - s))
|
|
69
|
+
return 0
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _cmd_pairs(args) -> int:
|
|
73
|
+
names = [os.path.basename(p) for p in args.images]
|
|
74
|
+
images = [load_image(p) for p in args.images]
|
|
75
|
+
cands = analyze_pairs(
|
|
76
|
+
images, names, page=args.page, cipher_entropy=args.cipher_entropy,
|
|
77
|
+
entropy_gap=args.entropy_gap, merge_gap=args.merge_gap)
|
|
78
|
+
if args.json:
|
|
79
|
+
print(json.dumps(cands, indent=2, ensure_ascii=False))
|
|
80
|
+
return 0
|
|
81
|
+
if not cands:
|
|
82
|
+
print("暗文↔平文対候補なし")
|
|
83
|
+
return 0
|
|
84
|
+
print("%d 候補:" % len(cands))
|
|
85
|
+
for c in cands[:args.limit]:
|
|
86
|
+
print("0x%08x-0x%08x (%d pages, basis=%s)"
|
|
87
|
+
% (c["start"], c["end"], c["pages"], c["basis"]))
|
|
88
|
+
print(" cipher=%s (ent %.2f) plain=%s (ent %.2f, %s)"
|
|
89
|
+
% (c["cipher_name"], c["cipher_entropy"],
|
|
90
|
+
c["plain_name"], c["plain_entropy"], c["plain_kind"]))
|
|
91
|
+
print(" refined: 0x%08x-0x%08x"
|
|
92
|
+
% (c["refined_start"], c["refined_end"]))
|
|
93
|
+
print(" desc (推定): %s" % c["descriptor"])
|
|
94
|
+
if len(cands) > args.limit:
|
|
95
|
+
print(" ... ほか %d 候補" % (len(cands) - args.limit))
|
|
96
|
+
return 0
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _cmd_triage(args) -> int:
|
|
100
|
+
info = triage(args.images, page=args.page,
|
|
101
|
+
cipher_entropy=args.cipher_entropy)
|
|
102
|
+
if args.json:
|
|
103
|
+
print(json.dumps(info, indent=2, ensure_ascii=False))
|
|
104
|
+
return 0
|
|
105
|
+
print("トリアージ: %d イメージ (page=0x%x):"
|
|
106
|
+
% (len(info["images"]), args.page))
|
|
107
|
+
for im in info["images"]:
|
|
108
|
+
print("%s: %d bytes entropy=%.2f"
|
|
109
|
+
% (im["name"], im["size"], im["entropy"]))
|
|
110
|
+
if im["region"]:
|
|
111
|
+
r = im["region"]
|
|
112
|
+
addr = " addr=0x%x" % r["addr"] if "addr" in r else ""
|
|
113
|
+
size = " size=%d" % r["size"] if "size" in r else ""
|
|
114
|
+
print(" REGION:%s%s (%s)" % (addr, size, r["text"]))
|
|
115
|
+
pe = im["pe"]
|
|
116
|
+
if pe:
|
|
117
|
+
secs = ", ".join(s["name"] for s in pe["sections"]) or "-"
|
|
118
|
+
print(" PE: %s %s @0x%x image_base=0x%x SizeOfImage=%s"
|
|
119
|
+
% (pe["bitness"], pe["machine"], pe["offset"],
|
|
120
|
+
pe["image_base"],
|
|
121
|
+
"0x%x" % pe["size_of_image"] if pe["size_of_image"]
|
|
122
|
+
else "?"))
|
|
123
|
+
print(" セクション: %s / レイアウト: %s%s"
|
|
124
|
+
% (secs, pe["layout"],
|
|
125
|
+
"" if pe["coverage"] is None
|
|
126
|
+
else " / カバレッジ: %.0f%%" % (pe["coverage"] * 100)))
|
|
127
|
+
c = im["census"]
|
|
128
|
+
if c:
|
|
129
|
+
print(" ページ: " + " / ".join("%s %d" % (k, c[k]) for k in c))
|
|
130
|
+
else:
|
|
131
|
+
print(" ページ: なし")
|
|
132
|
+
if "readable" in info:
|
|
133
|
+
print("読みやすい資産の分布 (上位 %d):" % args.limit)
|
|
134
|
+
for r in info["readable"][:args.limit]:
|
|
135
|
+
print(" 0x%08x-0x%08x %-16s %s (ent %.2f)"
|
|
136
|
+
% (r["start"], r["end"], r["source"], r["kind"],
|
|
137
|
+
r["entropy"]))
|
|
138
|
+
print("資産ランキング (平文ページ数順):")
|
|
139
|
+
for rank, s in enumerate(info["ranking"], 1):
|
|
140
|
+
print(" %d. %-16s plain=%d data=%d cipher=%d zero=%d"
|
|
141
|
+
% (rank, s["name"], s["plain_pages"], s["data_pages"],
|
|
142
|
+
s["cipher_pages"], s["zero_pages"]))
|
|
143
|
+
return 0
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _cmd_report(args) -> int:
|
|
147
|
+
if args.json:
|
|
148
|
+
out = json_report(
|
|
149
|
+
args.images, page=args.page, merge_gap=args.merge_gap,
|
|
150
|
+
threshold=args.threshold, top=args.top)
|
|
151
|
+
else:
|
|
152
|
+
out = markdown_report(
|
|
153
|
+
args.images, page=args.page, merge_gap=args.merge_gap,
|
|
154
|
+
threshold=args.threshold, top=args.top)
|
|
155
|
+
if args.out:
|
|
156
|
+
with open(args.out, "w", encoding="utf-8") as fh:
|
|
157
|
+
fh.write(out)
|
|
158
|
+
print("%s に書き出しました (%d bytes)" % (args.out, len(out)))
|
|
159
|
+
else:
|
|
160
|
+
print(out)
|
|
161
|
+
return 0
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def main(argv=None) -> int:
|
|
165
|
+
p = argparse.ArgumentParser(
|
|
166
|
+
prog="izayoi",
|
|
167
|
+
description="複数時点メモリダンプ比較ツールキット")
|
|
168
|
+
p.add_argument("--version", action="version",
|
|
169
|
+
version="%(prog)s " + __version__)
|
|
170
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
171
|
+
|
|
172
|
+
sp = sub.add_parser("entropy", help="ページ単位エントロピーと高エントロピー領域")
|
|
173
|
+
sp.add_argument("image")
|
|
174
|
+
sp.add_argument("--page", type=_int, default=0x1000)
|
|
175
|
+
sp.add_argument("--threshold", type=float, default=7.0)
|
|
176
|
+
sp.set_defaults(func=_cmd_entropy)
|
|
177
|
+
|
|
178
|
+
sp = sub.add_parser("diff", help="2ダンプ間の差分ラン列挙")
|
|
179
|
+
sp.add_argument("a")
|
|
180
|
+
sp.add_argument("b")
|
|
181
|
+
sp.add_argument("--merge-gap", type=_int, default=0)
|
|
182
|
+
sp.add_argument("--min-length", type=_int, default=1)
|
|
183
|
+
sp.add_argument("--limit", type=_int, default=50)
|
|
184
|
+
sp.set_defaults(func=_cmd_diff)
|
|
185
|
+
|
|
186
|
+
sp = sub.add_parser("compare", help="2〜3ダンプのページ分類")
|
|
187
|
+
sp.add_argument("images", nargs="+")
|
|
188
|
+
sp.add_argument("--page", type=_int, default=0x1000)
|
|
189
|
+
sp.add_argument("--ranges", action="store_true",
|
|
190
|
+
help="ラベルごとの連続領域も表示")
|
|
191
|
+
sp.set_defaults(func=_cmd_compare)
|
|
192
|
+
|
|
193
|
+
sp = sub.add_parser("pairs", help="暗文↔平文ページ対の推定(記述子形式で出力)")
|
|
194
|
+
sp.add_argument("images", nargs="+")
|
|
195
|
+
sp.add_argument("--page", type=_int, default=0x1000)
|
|
196
|
+
sp.add_argument("--cipher-entropy", type=float, default=7.0)
|
|
197
|
+
sp.add_argument("--entropy-gap", type=float, default=0.5)
|
|
198
|
+
sp.add_argument("--merge-gap", type=_int, default=16)
|
|
199
|
+
sp.add_argument("--json", action="store_true")
|
|
200
|
+
sp.add_argument("--limit", type=_int, default=20)
|
|
201
|
+
sp.set_defaults(func=_cmd_pairs)
|
|
202
|
+
|
|
203
|
+
sp = sub.add_parser(
|
|
204
|
+
"triage", help="ダンプの初動鑑定(PE検出・ページ集計・読みやすい資産の所在)")
|
|
205
|
+
sp.add_argument("images", nargs="+")
|
|
206
|
+
sp.add_argument("--page", type=_int, default=0x1000)
|
|
207
|
+
sp.add_argument("--cipher-entropy", type=float, default=7.0)
|
|
208
|
+
sp.add_argument("--json", action="store_true")
|
|
209
|
+
sp.add_argument("--limit", type=_int, default=20)
|
|
210
|
+
sp.set_defaults(func=_cmd_triage)
|
|
211
|
+
|
|
212
|
+
sp = sub.add_parser("report", help="Markdown/JSON レポート生成")
|
|
213
|
+
sp.add_argument("images", nargs="+")
|
|
214
|
+
sp.add_argument("--out", default=None)
|
|
215
|
+
sp.add_argument("--json", action="store_true")
|
|
216
|
+
sp.add_argument("--page", type=_int, default=0x1000)
|
|
217
|
+
sp.add_argument("--merge-gap", type=_int, default=16)
|
|
218
|
+
sp.add_argument("--threshold", type=float, default=7.0)
|
|
219
|
+
sp.add_argument("--top", type=_int, default=10)
|
|
220
|
+
sp.set_defaults(func=_cmd_report)
|
|
221
|
+
|
|
222
|
+
args = p.parse_args(argv)
|
|
223
|
+
return args.func(args)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
if __name__ == "__main__":
|
|
227
|
+
sys.exit(main())
|
izayoi/diffruns.py
ADDED
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
"""ダンプ間の差分ラン列挙とページ分類。"""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import hashlib
|
|
5
|
+
|
|
6
|
+
from . import fastpath
|
|
7
|
+
|
|
8
|
+
DEFAULT_PAGE = 0x1000
|
|
9
|
+
|
|
10
|
+
# ページ分類ラベル:
|
|
11
|
+
# static 全時点一致 … 静的領域(暗号文も含めて安定)
|
|
12
|
+
# flip 時点1==時点3 かつ時点2が異なる … 一時的にだけ内容が変わる
|
|
13
|
+
# (復号→実行→復元の候補)
|
|
14
|
+
# settled 時点1≠時点2==時点3 … 遷移後の状態書込
|
|
15
|
+
# volatile それ以外(全不一致含む) … 状態書込・ヒープ等
|
|
16
|
+
LABELS = ("static", "flip", "settled", "volatile")
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def diff_runs(a: bytes, b: bytes, merge_gap: int = 0, min_length: int = 1):
|
|
20
|
+
"""a vs b の不一致連続領域(ラン)を列挙する。
|
|
21
|
+
|
|
22
|
+
返り値: [{"offset": int, "length": int, "kind": "diff"|"a-only"|"b-only"}]
|
|
23
|
+
kind=diff 両イメージ範囲内で内容が不一致の区間
|
|
24
|
+
kind=a-only / b-only 片方のイメージが尽きた以降の領域
|
|
25
|
+
merge_gap>0 のとき、gap バイト以内の一致区間で隔てられた diff ランを
|
|
26
|
+
1本に併合する(併合後の長さにはスキップされた一致バイトを含む)。
|
|
27
|
+
min_length 未満のランは除外する(併合後に適用)。
|
|
28
|
+
|
|
29
|
+
内部の 0x1000 バイトチャンク走査は高速化のためのもので、
|
|
30
|
+
ランをチャンク境界で分割しない(0x1000 境界を跨ぐ不一致は
|
|
31
|
+
1 本の連続ランとして返す)。
|
|
32
|
+
"""
|
|
33
|
+
if fastpath.HAVE_NUMPY:
|
|
34
|
+
runs = fastpath.diff_runs(a, b)
|
|
35
|
+
else:
|
|
36
|
+
runs = _diff_runs_pure(a, b)
|
|
37
|
+
runs = _merge(runs, merge_gap)
|
|
38
|
+
return [r for r in runs if r["length"] >= min_length]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _diff_runs_pure(a: bytes, b: bytes):
|
|
42
|
+
"""diff ラン列挙の純実装(NumPy が無い環境で使われる)。"""
|
|
43
|
+
runs: list[dict] = []
|
|
44
|
+
n = min(len(a), len(b))
|
|
45
|
+
off = 0
|
|
46
|
+
start = None
|
|
47
|
+
while off < n:
|
|
48
|
+
end = min(off + 0x1000, n)
|
|
49
|
+
if a[off:end] != b[off:end]:
|
|
50
|
+
for i in range(off, end):
|
|
51
|
+
if a[i] != b[i]:
|
|
52
|
+
if start is None:
|
|
53
|
+
start = i
|
|
54
|
+
elif start is not None:
|
|
55
|
+
runs.append({"offset": start, "length": i - start,
|
|
56
|
+
"kind": "diff"})
|
|
57
|
+
start = None
|
|
58
|
+
elif start is not None:
|
|
59
|
+
# チャンク全体が一致 → 開いているランはここで閉じる
|
|
60
|
+
runs.append({"offset": start, "length": off - start,
|
|
61
|
+
"kind": "diff"})
|
|
62
|
+
start = None
|
|
63
|
+
off = end
|
|
64
|
+
if start is not None:
|
|
65
|
+
runs.append({"offset": start, "length": n - start, "kind": "diff"})
|
|
66
|
+
if len(a) > len(b):
|
|
67
|
+
runs.append({"offset": n, "length": len(a) - n, "kind": "a-only"})
|
|
68
|
+
elif len(b) > len(a):
|
|
69
|
+
runs.append({"offset": n, "length": len(b) - n, "kind": "b-only"})
|
|
70
|
+
return runs
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _merge(runs: list[dict], gap: int) -> list[dict]:
|
|
74
|
+
if gap <= 0:
|
|
75
|
+
return runs
|
|
76
|
+
out: list[dict] = []
|
|
77
|
+
for r in runs:
|
|
78
|
+
if out and r["kind"] == "diff" and out[-1]["kind"] == "diff":
|
|
79
|
+
prev_end = out[-1]["offset"] + out[-1]["length"]
|
|
80
|
+
if r["offset"] - prev_end <= gap:
|
|
81
|
+
out[-1]["length"] = r["offset"] + r["length"] - out[-1]["offset"]
|
|
82
|
+
continue
|
|
83
|
+
out.append(dict(r))
|
|
84
|
+
return out
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _page_hashes(data: bytes, page: int):
|
|
88
|
+
return [hashlib.md5(data[o:o + page]).digest()
|
|
89
|
+
for o in range(0, len(data), page)]
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def classify_pages(images, page: int = DEFAULT_PAGE):
|
|
93
|
+
"""2〜3イメージのページ単位分類。
|
|
94
|
+
|
|
95
|
+
返り値: [(offset, label)] — 全イメージの共通ページ範囲のみ
|
|
96
|
+
(長さが異なるイメージの超出ページは無視)。
|
|
97
|
+
2イメージの場合は static / volatile のみを返す。
|
|
98
|
+
"""
|
|
99
|
+
if len(images) < 2:
|
|
100
|
+
raise ValueError("2つ以上のイメージが必要です")
|
|
101
|
+
if len(images) > 3:
|
|
102
|
+
raise ValueError("比較できるのは3イメージまでです(ラベル定義の制約)")
|
|
103
|
+
if fastpath.HAVE_NUMPY:
|
|
104
|
+
return fastpath.classify_pages(images, page)
|
|
105
|
+
hashes = [_page_hashes(img, page) for img in images]
|
|
106
|
+
npages = min(len(h) for h in hashes)
|
|
107
|
+
out = []
|
|
108
|
+
for i in range(npages):
|
|
109
|
+
hs = [h[i] for h in hashes]
|
|
110
|
+
if all(x == hs[0] for x in hs):
|
|
111
|
+
label = "static"
|
|
112
|
+
elif len(hs) == 3 and hs[0] == hs[2]:
|
|
113
|
+
label = "flip"
|
|
114
|
+
elif len(hs) == 3 and hs[1] == hs[2]:
|
|
115
|
+
label = "settled"
|
|
116
|
+
else:
|
|
117
|
+
label = "volatile"
|
|
118
|
+
out.append((i * page, label))
|
|
119
|
+
return out
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def summarize(items, page: int = DEFAULT_PAGE):
|
|
123
|
+
"""[(offset, label)] から {label: (pages, bytes)} へ集計。"""
|
|
124
|
+
acc = {}
|
|
125
|
+
for _, label in items:
|
|
126
|
+
p, b = acc.get(label, (0, 0))
|
|
127
|
+
acc[label] = (p + 1, b + page)
|
|
128
|
+
return acc
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def label_ranges(items, page: int = DEFAULT_PAGE):
|
|
132
|
+
"""隣接する同一ラベルを連続領域に併合する。
|
|
133
|
+
|
|
134
|
+
返り値: {label: [(start, end), ...]} (end は排他)。
|
|
135
|
+
"""
|
|
136
|
+
out = {}
|
|
137
|
+
for off, label in items:
|
|
138
|
+
ranges = out.setdefault(label, [])
|
|
139
|
+
if ranges and ranges[-1][1] == off:
|
|
140
|
+
ranges[-1] = (ranges[-1][0], off + page)
|
|
141
|
+
else:
|
|
142
|
+
ranges.append((off, off + page))
|
|
143
|
+
return out
|
izayoi/entropy.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""画像のエントロピー分析(全体およびページ単位)。"""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import math
|
|
5
|
+
from collections import Counter
|
|
6
|
+
|
|
7
|
+
from . import fastpath
|
|
8
|
+
|
|
9
|
+
DEFAULT_PAGE = 0x1000
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def shannon_entropy(data: bytes) -> float:
|
|
13
|
+
"""バイト列のシャノンエントロピー(ビット/バイト、0.0〜8.0)。"""
|
|
14
|
+
if fastpath.HAVE_NUMPY:
|
|
15
|
+
return fastpath.shannon_entropy(data)
|
|
16
|
+
if not data:
|
|
17
|
+
return 0.0
|
|
18
|
+
counts = Counter(data)
|
|
19
|
+
n = len(data)
|
|
20
|
+
return -sum((v / n) * math.log2(v / n) for v in counts.values())
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def entropy_map(data: bytes, page: int = DEFAULT_PAGE):
|
|
24
|
+
"""ページ単位のエントロピー。
|
|
25
|
+
|
|
26
|
+
返り値: [(offset, entropy, chunk_size), ...] (最終ページは端数になる)。
|
|
27
|
+
"""
|
|
28
|
+
if fastpath.HAVE_NUMPY:
|
|
29
|
+
return fastpath.entropy_map(data, page)
|
|
30
|
+
out = []
|
|
31
|
+
for off in range(0, len(data), page):
|
|
32
|
+
chunk = data[off:off + page]
|
|
33
|
+
out.append((off, shannon_entropy(chunk), len(chunk)))
|
|
34
|
+
return out
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def high_entropy_ranges(data: bytes, threshold: float = 7.0,
|
|
38
|
+
page: int = DEFAULT_PAGE):
|
|
39
|
+
"""threshold 以上のエントロピーを持つページの連続領域を列挙する。
|
|
40
|
+
|
|
41
|
+
暗号化・圧縮領域の候補探しに使う。
|
|
42
|
+
返り値: [(start, end, max_entropy), ...] (end は排他)。
|
|
43
|
+
"""
|
|
44
|
+
ranges = []
|
|
45
|
+
cur = None
|
|
46
|
+
for off, ent, size in entropy_map(data, page):
|
|
47
|
+
if ent >= threshold:
|
|
48
|
+
if cur is None:
|
|
49
|
+
cur = [off, off + size, ent]
|
|
50
|
+
else:
|
|
51
|
+
cur[1] = off + size
|
|
52
|
+
if ent > cur[2]:
|
|
53
|
+
cur[2] = ent
|
|
54
|
+
elif cur is not None:
|
|
55
|
+
ranges.append(tuple(cur))
|
|
56
|
+
cur = None
|
|
57
|
+
if cur is not None:
|
|
58
|
+
ranges.append(tuple(cur))
|
|
59
|
+
return ranges
|