izayoi 0.5.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
izayoi/__init__.py ADDED
@@ -0,0 +1,29 @@
1
+ """izayoi — 複数時点メモリダンプ比較ツールキット。
2
+
3
+ ページ単位のエントロピー分析、ダンプ間の差分ラン列挙、
4
+ 複数時点ダンプのページ分類(static/flip/settled/volatile)、
5
+ 暗文↔平文ページ対の推定、ダンプのトリアージ
6
+ (PEヘッダ検出・読みやすい資産の所在推定)を提供する。
7
+ 標準ライブラリのみで動作し、NumPy が導入されていれば
8
+ ホットパスが自動的にベクトル化実装に切り替わる(fastpath)。
9
+ """
10
+ from .diffruns import classify_pages, diff_runs, label_ranges, summarize
11
+ from .entropy import entropy_map, high_entropy_ranges, shannon_entropy
12
+ from .images import fingerprint, load_image, strip_header
13
+ from .pairs import (analyze_pairs, code_density, descriptor_suggestion,
14
+ find_pairs, page_class, printable_ratio,
15
+ refine_with_diffs, zero_ratio)
16
+ from .triage import (detect_pe, page_census, readability_ranking,
17
+ readable_sources, region_info, triage, triage_images)
18
+
19
+ __version__ = "0.5.2"
20
+
21
+ __all__ = [
22
+ "analyze_pairs", "classify_pages", "code_density", "detect_pe",
23
+ "descriptor_suggestion", "diff_runs", "entropy_map", "fingerprint",
24
+ "find_pairs", "high_entropy_ranges", "label_ranges", "load_image",
25
+ "page_class", "page_census", "printable_ratio", "readability_ranking",
26
+ "readable_sources", "refine_with_diffs", "region_info", "shannon_entropy",
27
+ "strip_header", "summarize", "triage", "triage_images", "zero_ratio",
28
+ "__version__",
29
+ ]
izayoi/cli.py ADDED
@@ -0,0 +1,227 @@
1
+ """izayoi コマンドライン。
2
+
3
+ 使い方: python -m izayoi.cli <cmd> ...
4
+ """
5
+ from __future__ import annotations
6
+
7
+ import argparse
8
+ import json
9
+ import os
10
+ import sys
11
+
12
+ from . import __version__
13
+
14
+ from .diffruns import classify_pages, diff_runs, label_ranges, summarize
15
+ from .entropy import high_entropy_ranges
16
+ from .images import load_image
17
+ from .pairs import analyze_pairs
18
+ from .report import json_report, markdown_report
19
+ from .triage import triage
20
+
21
+
22
+ def _int(s: str) -> int:
23
+ return int(s, 0)
24
+
25
+
26
+ def _cmd_entropy(args) -> int:
27
+ data = load_image(args.image)
28
+ ranges = high_entropy_ranges(data, args.threshold, args.page)
29
+ if not ranges:
30
+ print("高エントロピー領域なし (threshold=%.2f)" % args.threshold)
31
+ return 0
32
+ print("%d 領域 (threshold=%.2f, page=0x%x):"
33
+ % (len(ranges), args.threshold, args.page))
34
+ for start, end, mx in ranges:
35
+ print(" 0x%08x-0x%08x %6d bytes max=%.2f"
36
+ % (start, end, end - start, mx))
37
+ return 0
38
+
39
+
40
+ def _cmd_diff(args) -> int:
41
+ a, b = load_image(args.a), load_image(args.b)
42
+ runs = diff_runs(a, b, merge_gap=args.merge_gap, min_length=args.min_length)
43
+ if not runs:
44
+ print("差分なし")
45
+ return 0
46
+ total = sum(r["length"] for r in runs)
47
+ print("%d ラン / %d bytes (merge_gap=%d, min_length=%d):"
48
+ % (len(runs), total, args.merge_gap, args.min_length))
49
+ for r in runs[:args.limit]:
50
+ print(" 0x%08x %8d %s" % (r["offset"], r["length"], r["kind"]))
51
+ if len(runs) > args.limit:
52
+ print(" ... ほか %d ラン" % (len(runs) - args.limit))
53
+ return 0
54
+
55
+
56
+ def _cmd_compare(args) -> int:
57
+ images = [load_image(p) for p in args.images]
58
+ items = classify_pages(images, args.page)
59
+ acc = summarize(items, args.page)
60
+ print("ページ分類 (page=0x%x):" % args.page)
61
+ for label, (npages, nbytes) in acc.items():
62
+ print(" %-8s %5d ページ / %8d bytes" % (label, npages, nbytes))
63
+ if args.ranges:
64
+ print("領域一覧:")
65
+ for label, ranges in label_ranges(items, args.page).items():
66
+ for s, e in ranges:
67
+ print(" %-8s 0x%08x-0x%08x (%d bytes)"
68
+ % (label, s, e, e - s))
69
+ return 0
70
+
71
+
72
+ def _cmd_pairs(args) -> int:
73
+ names = [os.path.basename(p) for p in args.images]
74
+ images = [load_image(p) for p in args.images]
75
+ cands = analyze_pairs(
76
+ images, names, page=args.page, cipher_entropy=args.cipher_entropy,
77
+ entropy_gap=args.entropy_gap, merge_gap=args.merge_gap)
78
+ if args.json:
79
+ print(json.dumps(cands, indent=2, ensure_ascii=False))
80
+ return 0
81
+ if not cands:
82
+ print("暗文↔平文対候補なし")
83
+ return 0
84
+ print("%d 候補:" % len(cands))
85
+ for c in cands[:args.limit]:
86
+ print("0x%08x-0x%08x (%d pages, basis=%s)"
87
+ % (c["start"], c["end"], c["pages"], c["basis"]))
88
+ print(" cipher=%s (ent %.2f) plain=%s (ent %.2f, %s)"
89
+ % (c["cipher_name"], c["cipher_entropy"],
90
+ c["plain_name"], c["plain_entropy"], c["plain_kind"]))
91
+ print(" refined: 0x%08x-0x%08x"
92
+ % (c["refined_start"], c["refined_end"]))
93
+ print(" desc (推定): %s" % c["descriptor"])
94
+ if len(cands) > args.limit:
95
+ print(" ... ほか %d 候補" % (len(cands) - args.limit))
96
+ return 0
97
+
98
+
99
+ def _cmd_triage(args) -> int:
100
+ info = triage(args.images, page=args.page,
101
+ cipher_entropy=args.cipher_entropy)
102
+ if args.json:
103
+ print(json.dumps(info, indent=2, ensure_ascii=False))
104
+ return 0
105
+ print("トリアージ: %d イメージ (page=0x%x):"
106
+ % (len(info["images"]), args.page))
107
+ for im in info["images"]:
108
+ print("%s: %d bytes entropy=%.2f"
109
+ % (im["name"], im["size"], im["entropy"]))
110
+ if im["region"]:
111
+ r = im["region"]
112
+ addr = " addr=0x%x" % r["addr"] if "addr" in r else ""
113
+ size = " size=%d" % r["size"] if "size" in r else ""
114
+ print(" REGION:%s%s (%s)" % (addr, size, r["text"]))
115
+ pe = im["pe"]
116
+ if pe:
117
+ secs = ", ".join(s["name"] for s in pe["sections"]) or "-"
118
+ print(" PE: %s %s @0x%x image_base=0x%x SizeOfImage=%s"
119
+ % (pe["bitness"], pe["machine"], pe["offset"],
120
+ pe["image_base"],
121
+ "0x%x" % pe["size_of_image"] if pe["size_of_image"]
122
+ else "?"))
123
+ print(" セクション: %s / レイアウト: %s%s"
124
+ % (secs, pe["layout"],
125
+ "" if pe["coverage"] is None
126
+ else " / カバレッジ: %.0f%%" % (pe["coverage"] * 100)))
127
+ c = im["census"]
128
+ if c:
129
+ print(" ページ: " + " / ".join("%s %d" % (k, c[k]) for k in c))
130
+ else:
131
+ print(" ページ: なし")
132
+ if "readable" in info:
133
+ print("読みやすい資産の分布 (上位 %d):" % args.limit)
134
+ for r in info["readable"][:args.limit]:
135
+ print(" 0x%08x-0x%08x %-16s %s (ent %.2f)"
136
+ % (r["start"], r["end"], r["source"], r["kind"],
137
+ r["entropy"]))
138
+ print("資産ランキング (平文ページ数順):")
139
+ for rank, s in enumerate(info["ranking"], 1):
140
+ print(" %d. %-16s plain=%d data=%d cipher=%d zero=%d"
141
+ % (rank, s["name"], s["plain_pages"], s["data_pages"],
142
+ s["cipher_pages"], s["zero_pages"]))
143
+ return 0
144
+
145
+
146
+ def _cmd_report(args) -> int:
147
+ if args.json:
148
+ out = json_report(
149
+ args.images, page=args.page, merge_gap=args.merge_gap,
150
+ threshold=args.threshold, top=args.top)
151
+ else:
152
+ out = markdown_report(
153
+ args.images, page=args.page, merge_gap=args.merge_gap,
154
+ threshold=args.threshold, top=args.top)
155
+ if args.out:
156
+ with open(args.out, "w", encoding="utf-8") as fh:
157
+ fh.write(out)
158
+ print("%s に書き出しました (%d bytes)" % (args.out, len(out)))
159
+ else:
160
+ print(out)
161
+ return 0
162
+
163
+
164
+ def main(argv=None) -> int:
165
+ p = argparse.ArgumentParser(
166
+ prog="izayoi",
167
+ description="複数時点メモリダンプ比較ツールキット")
168
+ p.add_argument("--version", action="version",
169
+ version="%(prog)s " + __version__)
170
+ sub = p.add_subparsers(dest="cmd", required=True)
171
+
172
+ sp = sub.add_parser("entropy", help="ページ単位エントロピーと高エントロピー領域")
173
+ sp.add_argument("image")
174
+ sp.add_argument("--page", type=_int, default=0x1000)
175
+ sp.add_argument("--threshold", type=float, default=7.0)
176
+ sp.set_defaults(func=_cmd_entropy)
177
+
178
+ sp = sub.add_parser("diff", help="2ダンプ間の差分ラン列挙")
179
+ sp.add_argument("a")
180
+ sp.add_argument("b")
181
+ sp.add_argument("--merge-gap", type=_int, default=0)
182
+ sp.add_argument("--min-length", type=_int, default=1)
183
+ sp.add_argument("--limit", type=_int, default=50)
184
+ sp.set_defaults(func=_cmd_diff)
185
+
186
+ sp = sub.add_parser("compare", help="2〜3ダンプのページ分類")
187
+ sp.add_argument("images", nargs="+")
188
+ sp.add_argument("--page", type=_int, default=0x1000)
189
+ sp.add_argument("--ranges", action="store_true",
190
+ help="ラベルごとの連続領域も表示")
191
+ sp.set_defaults(func=_cmd_compare)
192
+
193
+ sp = sub.add_parser("pairs", help="暗文↔平文ページ対の推定(記述子形式で出力)")
194
+ sp.add_argument("images", nargs="+")
195
+ sp.add_argument("--page", type=_int, default=0x1000)
196
+ sp.add_argument("--cipher-entropy", type=float, default=7.0)
197
+ sp.add_argument("--entropy-gap", type=float, default=0.5)
198
+ sp.add_argument("--merge-gap", type=_int, default=16)
199
+ sp.add_argument("--json", action="store_true")
200
+ sp.add_argument("--limit", type=_int, default=20)
201
+ sp.set_defaults(func=_cmd_pairs)
202
+
203
+ sp = sub.add_parser(
204
+ "triage", help="ダンプの初動鑑定(PE検出・ページ集計・読みやすい資産の所在)")
205
+ sp.add_argument("images", nargs="+")
206
+ sp.add_argument("--page", type=_int, default=0x1000)
207
+ sp.add_argument("--cipher-entropy", type=float, default=7.0)
208
+ sp.add_argument("--json", action="store_true")
209
+ sp.add_argument("--limit", type=_int, default=20)
210
+ sp.set_defaults(func=_cmd_triage)
211
+
212
+ sp = sub.add_parser("report", help="Markdown/JSON レポート生成")
213
+ sp.add_argument("images", nargs="+")
214
+ sp.add_argument("--out", default=None)
215
+ sp.add_argument("--json", action="store_true")
216
+ sp.add_argument("--page", type=_int, default=0x1000)
217
+ sp.add_argument("--merge-gap", type=_int, default=16)
218
+ sp.add_argument("--threshold", type=float, default=7.0)
219
+ sp.add_argument("--top", type=_int, default=10)
220
+ sp.set_defaults(func=_cmd_report)
221
+
222
+ args = p.parse_args(argv)
223
+ return args.func(args)
224
+
225
+
226
+ if __name__ == "__main__":
227
+ sys.exit(main())
izayoi/diffruns.py ADDED
@@ -0,0 +1,143 @@
1
+ """ダンプ間の差分ラン列挙とページ分類。"""
2
+ from __future__ import annotations
3
+
4
+ import hashlib
5
+
6
+ from . import fastpath
7
+
8
+ DEFAULT_PAGE = 0x1000
9
+
10
+ # ページ分類ラベル:
11
+ # static 全時点一致 … 静的領域(暗号文も含めて安定)
12
+ # flip 時点1==時点3 かつ時点2が異なる … 一時的にだけ内容が変わる
13
+ # (復号→実行→復元の候補)
14
+ # settled 時点1≠時点2==時点3 … 遷移後の状態書込
15
+ # volatile それ以外(全不一致含む) … 状態書込・ヒープ等
16
+ LABELS = ("static", "flip", "settled", "volatile")
17
+
18
+
19
+ def diff_runs(a: bytes, b: bytes, merge_gap: int = 0, min_length: int = 1):
20
+ """a vs b の不一致連続領域(ラン)を列挙する。
21
+
22
+ 返り値: [{"offset": int, "length": int, "kind": "diff"|"a-only"|"b-only"}]
23
+ kind=diff 両イメージ範囲内で内容が不一致の区間
24
+ kind=a-only / b-only 片方のイメージが尽きた以降の領域
25
+ merge_gap>0 のとき、gap バイト以内の一致区間で隔てられた diff ランを
26
+ 1本に併合する(併合後の長さにはスキップされた一致バイトを含む)。
27
+ min_length 未満のランは除外する(併合後に適用)。
28
+
29
+ 内部の 0x1000 バイトチャンク走査は高速化のためのもので、
30
+ ランをチャンク境界で分割しない(0x1000 境界を跨ぐ不一致は
31
+ 1 本の連続ランとして返す)。
32
+ """
33
+ if fastpath.HAVE_NUMPY:
34
+ runs = fastpath.diff_runs(a, b)
35
+ else:
36
+ runs = _diff_runs_pure(a, b)
37
+ runs = _merge(runs, merge_gap)
38
+ return [r for r in runs if r["length"] >= min_length]
39
+
40
+
41
+ def _diff_runs_pure(a: bytes, b: bytes):
42
+ """diff ラン列挙の純実装(NumPy が無い環境で使われる)。"""
43
+ runs: list[dict] = []
44
+ n = min(len(a), len(b))
45
+ off = 0
46
+ start = None
47
+ while off < n:
48
+ end = min(off + 0x1000, n)
49
+ if a[off:end] != b[off:end]:
50
+ for i in range(off, end):
51
+ if a[i] != b[i]:
52
+ if start is None:
53
+ start = i
54
+ elif start is not None:
55
+ runs.append({"offset": start, "length": i - start,
56
+ "kind": "diff"})
57
+ start = None
58
+ elif start is not None:
59
+ # チャンク全体が一致 → 開いているランはここで閉じる
60
+ runs.append({"offset": start, "length": off - start,
61
+ "kind": "diff"})
62
+ start = None
63
+ off = end
64
+ if start is not None:
65
+ runs.append({"offset": start, "length": n - start, "kind": "diff"})
66
+ if len(a) > len(b):
67
+ runs.append({"offset": n, "length": len(a) - n, "kind": "a-only"})
68
+ elif len(b) > len(a):
69
+ runs.append({"offset": n, "length": len(b) - n, "kind": "b-only"})
70
+ return runs
71
+
72
+
73
+ def _merge(runs: list[dict], gap: int) -> list[dict]:
74
+ if gap <= 0:
75
+ return runs
76
+ out: list[dict] = []
77
+ for r in runs:
78
+ if out and r["kind"] == "diff" and out[-1]["kind"] == "diff":
79
+ prev_end = out[-1]["offset"] + out[-1]["length"]
80
+ if r["offset"] - prev_end <= gap:
81
+ out[-1]["length"] = r["offset"] + r["length"] - out[-1]["offset"]
82
+ continue
83
+ out.append(dict(r))
84
+ return out
85
+
86
+
87
+ def _page_hashes(data: bytes, page: int):
88
+ return [hashlib.md5(data[o:o + page]).digest()
89
+ for o in range(0, len(data), page)]
90
+
91
+
92
+ def classify_pages(images, page: int = DEFAULT_PAGE):
93
+ """2〜3イメージのページ単位分類。
94
+
95
+ 返り値: [(offset, label)] — 全イメージの共通ページ範囲のみ
96
+ (長さが異なるイメージの超出ページは無視)。
97
+ 2イメージの場合は static / volatile のみを返す。
98
+ """
99
+ if len(images) < 2:
100
+ raise ValueError("2つ以上のイメージが必要です")
101
+ if len(images) > 3:
102
+ raise ValueError("比較できるのは3イメージまでです(ラベル定義の制約)")
103
+ if fastpath.HAVE_NUMPY:
104
+ return fastpath.classify_pages(images, page)
105
+ hashes = [_page_hashes(img, page) for img in images]
106
+ npages = min(len(h) for h in hashes)
107
+ out = []
108
+ for i in range(npages):
109
+ hs = [h[i] for h in hashes]
110
+ if all(x == hs[0] for x in hs):
111
+ label = "static"
112
+ elif len(hs) == 3 and hs[0] == hs[2]:
113
+ label = "flip"
114
+ elif len(hs) == 3 and hs[1] == hs[2]:
115
+ label = "settled"
116
+ else:
117
+ label = "volatile"
118
+ out.append((i * page, label))
119
+ return out
120
+
121
+
122
+ def summarize(items, page: int = DEFAULT_PAGE):
123
+ """[(offset, label)] から {label: (pages, bytes)} へ集計。"""
124
+ acc = {}
125
+ for _, label in items:
126
+ p, b = acc.get(label, (0, 0))
127
+ acc[label] = (p + 1, b + page)
128
+ return acc
129
+
130
+
131
+ def label_ranges(items, page: int = DEFAULT_PAGE):
132
+ """隣接する同一ラベルを連続領域に併合する。
133
+
134
+ 返り値: {label: [(start, end), ...]} (end は排他)。
135
+ """
136
+ out = {}
137
+ for off, label in items:
138
+ ranges = out.setdefault(label, [])
139
+ if ranges and ranges[-1][1] == off:
140
+ ranges[-1] = (ranges[-1][0], off + page)
141
+ else:
142
+ ranges.append((off, off + page))
143
+ return out
izayoi/entropy.py ADDED
@@ -0,0 +1,59 @@
1
+ """画像のエントロピー分析(全体およびページ単位)。"""
2
+ from __future__ import annotations
3
+
4
+ import math
5
+ from collections import Counter
6
+
7
+ from . import fastpath
8
+
9
+ DEFAULT_PAGE = 0x1000
10
+
11
+
12
+ def shannon_entropy(data: bytes) -> float:
13
+ """バイト列のシャノンエントロピー(ビット/バイト、0.0〜8.0)。"""
14
+ if fastpath.HAVE_NUMPY:
15
+ return fastpath.shannon_entropy(data)
16
+ if not data:
17
+ return 0.0
18
+ counts = Counter(data)
19
+ n = len(data)
20
+ return -sum((v / n) * math.log2(v / n) for v in counts.values())
21
+
22
+
23
+ def entropy_map(data: bytes, page: int = DEFAULT_PAGE):
24
+ """ページ単位のエントロピー。
25
+
26
+ 返り値: [(offset, entropy, chunk_size), ...] (最終ページは端数になる)。
27
+ """
28
+ if fastpath.HAVE_NUMPY:
29
+ return fastpath.entropy_map(data, page)
30
+ out = []
31
+ for off in range(0, len(data), page):
32
+ chunk = data[off:off + page]
33
+ out.append((off, shannon_entropy(chunk), len(chunk)))
34
+ return out
35
+
36
+
37
+ def high_entropy_ranges(data: bytes, threshold: float = 7.0,
38
+ page: int = DEFAULT_PAGE):
39
+ """threshold 以上のエントロピーを持つページの連続領域を列挙する。
40
+
41
+ 暗号化・圧縮領域の候補探しに使う。
42
+ 返り値: [(start, end, max_entropy), ...] (end は排他)。
43
+ """
44
+ ranges = []
45
+ cur = None
46
+ for off, ent, size in entropy_map(data, page):
47
+ if ent >= threshold:
48
+ if cur is None:
49
+ cur = [off, off + size, ent]
50
+ else:
51
+ cur[1] = off + size
52
+ if ent > cur[2]:
53
+ cur[2] = ent
54
+ elif cur is not None:
55
+ ranges.append(tuple(cur))
56
+ cur = None
57
+ if cur is not None:
58
+ ranges.append(tuple(cur))
59
+ return ranges