apkdiff-core 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
apkdiff_core/dex.py ADDED
@@ -0,0 +1,900 @@
1
+ """DEX parser: attribute every byte of a dex file to a concrete class.
2
+
3
+ This is not a full dex decompiler. It answers one question:
4
+ **"How many bytes does this class cost?"**
5
+
6
+ Classes live in classes*.dex. A dex is a dense set of tables and structures with
7
+ no per-class byte count, so we compute one.
8
+
9
+ How it works:
10
+ Some blocks clearly belong to a class (its method bytecode); other blocks are
11
+ shared by all classes (string pool, type table).
12
+
13
+ 1. Charge class-specific blocks directly to that class.
14
+ 2. Charge shared blocks (for example a string) to the first class that
15
+ references them. The order is fixed, so results are deterministic.
16
+ 3. Put everything else (header, padding, unreferenced strings) into
17
+ `unattributed`.
18
+
19
+ Therefore: sum(class bytes) + unattributed = dex file size. Tests enforce
20
+ this invariant.
21
+
22
+ What is charged to a class?
23
+ - class_def_item: the class record (fixed 32 bytes)
24
+ - class_data_item: field and method declarations
25
+ - code_item: bytecode + try/catch handlers per method
26
+ - debug_info_item: debug info (local names, line numbers)
27
+ - annotations on the class, fields and methods
28
+ - static_values: static field initial values
29
+ - referenced identifier-table and string-pool entries
30
+
31
+ Two key designs:
32
+
33
+ [Byte attribution] The accounting described above; see analyze().
34
+ [Semantic fingerprint] Change detection does NOT compare raw dex bytes.
35
+ Adding or removing any class renumbers every index, changing raw bytes for
36
+ classes whose source did not change. The fingerprint records only
37
+ source-meaningful data: class name, superclass, interfaces, field/method
38
+ signatures, local variable names, annotations and source file.
39
+
40
+ The size computed here is **uncompressed** dex bytes; apk.py estimates compression.
41
+ """
42
+
43
+ from __future__ import annotations
44
+
45
+ import hashlib
46
+ import struct
47
+ from collections import Counter
48
+ from dataclasses import dataclass, field
49
+
50
+ from .util import SizeBreakdown
51
+
52
+
53
+ class DexError(Exception):
54
+ pass
55
+
56
+
57
+ # encoded_value type tags shared by the size walker and the reference collector.
58
+ _PRIMITIVE_VALUE_TYPES = frozenset({0x00, 0x02, 0x03, 0x04, 0x05, 0x10, 0x11})
59
+ _INDEX_VALUE_TYPES = frozenset(range(0x15, 0x1C))
60
+
61
+
62
+ # ---------------------------------------------------------------------------
63
+ # Low-level helpers: byte readers.
64
+ # ---------------------------------------------------------------------------
65
+ # A dex is a byte stream. These helpers read integers at different widths:
66
+ # _u1 reads 1 byte, _u2 reads 2, _u4 reads 4.
67
+ # They raise on out-of-bounds reads instead of silently returning garbage.
68
+ # _uleb128 / _sleb128 are variable-length integers: small values use one
69
+ # byte, larger values use more. This is dex's space-saving encoding.
70
+
71
+
72
+ def _u1(data: bytes, off: int) -> int:
73
+ if off < 0 or off >= len(data):
74
+ raise DexError(f"u1 read out of bounds at 0x{off:x}")
75
+ return data[off]
76
+
77
+
78
+ def _u2(data: bytes, off: int) -> int:
79
+ if off < 0 or off + 2 > len(data):
80
+ raise DexError(f"u2 read out of bounds at 0x{off:x}")
81
+ return struct.unpack_from("<H", data, off)[0]
82
+
83
+
84
+ def _u4(data: bytes, off: int) -> int:
85
+ if off < 0 or off + 4 > len(data):
86
+ raise DexError(f"u4 read out of bounds at 0x{off:x}")
87
+ return struct.unpack_from("<I", data, off)[0]
88
+
89
+
90
+ def _uleb128(data: bytes, off: int) -> tuple[int, int]:
91
+ """Read an unsigned LEB128 value; returns (value, new offset).
92
+
93
+ LEB128 uses 7 data bits per byte; the high bit means "more bytes follow".
94
+ Small values (0-127) take one byte, larger values take more, saving space.
95
+ """
96
+ result = 0
97
+ shift = 0
98
+ while True:
99
+ b = _u1(data, off)
100
+ off += 1
101
+ result |= (b & 0x7F) << shift # Append the low 7 bits to the result.
102
+ if not b & 0x80: # High bit 0 means this is the final byte.
103
+ return result, off
104
+ shift += 7
105
+ if shift > 35:
106
+ raise DexError("uleb128 too long")
107
+
108
+
109
+ def _sleb128(data: bytes, off: int) -> tuple[int, int]:
110
+ result = 0
111
+ shift = 0
112
+ while True:
113
+ b = _u1(data, off)
114
+ off += 1
115
+ result |= (b & 0x7F) << shift
116
+ shift += 7
117
+ if not b & 0x80:
118
+ if b & 0x40:
119
+ result -= 1 << shift
120
+ return result, off
121
+ if shift > 63:
122
+ raise DexError("sleb128 too long")
123
+
124
+
125
+ def _decode_mutf8(raw: bytes) -> str:
126
+ """Decode DEX MUTF-8 (surrogate pairs folded back into codepoints)."""
127
+ units: list[int] = []
128
+ i = 0
129
+ while i < len(raw):
130
+ b = raw[i]
131
+ if b == 0:
132
+ break
133
+ if b < 0x80:
134
+ units.append(b)
135
+ i += 1
136
+ elif b & 0xE0 == 0xC0:
137
+ if i + 1 >= len(raw):
138
+ break
139
+ units.append(((b & 0x1F) << 6) | (raw[i + 1] & 0x3F))
140
+ i += 2
141
+ elif b & 0xF0 == 0xE0:
142
+ if i + 2 >= len(raw):
143
+ break
144
+ units.append(
145
+ ((b & 0x0F) << 12)
146
+ | ((raw[i + 1] & 0x3F) << 6)
147
+ | (raw[i + 2] & 0x3F)
148
+ )
149
+ i += 3
150
+ else:
151
+ units.append(0xFFFD)
152
+ i += 1
153
+
154
+ out: list[str] = []
155
+ j = 0
156
+ while j < len(units):
157
+ u = units[j]
158
+ if 0xD800 <= u <= 0xDBFF and j + 1 < len(units) and 0xDC00 <= units[j + 1] <= 0xDFFF:
159
+ low = units[j + 1]
160
+ cp = 0x10000 + ((u - 0xD800) << 10) + (low - 0xDC00)
161
+ out.append(chr(cp))
162
+ j += 2
163
+ else:
164
+ out.append(chr(u) if u <= 0x10FFFF else "\ufffd")
165
+ j += 1
166
+ return "".join(out)
167
+
168
+
169
+ def descriptor_to_class_name(desc: str) -> str:
170
+ """'Lcom/foo/Bar;' -> 'com.foo.Bar'; '$' is kept for inner classes."""
171
+ if desc.startswith("L") and desc.endswith(";"):
172
+ return desc[1:-1].replace("/", ".")
173
+ return desc
174
+
175
+
176
+ # ---------------------------------------------------------------------------
177
+ # data model
178
+ # ---------------------------------------------------------------------------
179
+
180
+
181
+ @dataclass
182
+ class DexClass:
183
+ name: str # descriptor, e.g. Lcom/foo/Bar;
184
+ dex_name: str # which dex file it came from
185
+ size: int
186
+ breakdown: SizeBreakdown
187
+ content_hash: str
188
+ methods: int = 0
189
+ fields: int = 0
190
+ partial: bool = False # some structures could not be parsed
191
+ warning: str = ""
192
+ semantic_parts: list[str] = field(default_factory=list)
193
+
194
+
195
+ @dataclass
196
+ class DexAnalysis:
197
+ path: str
198
+ file_size: int
199
+ classes: list[DexClass] = field(default_factory=list)
200
+ unattributed: int = 0
201
+ warnings: list[str] = field(default_factory=list)
202
+
203
+
204
+ # ---------------------------------------------------------------------------
205
+ # parser
206
+ # ---------------------------------------------------------------------------
207
+
208
+
209
+ class DexFile:
210
+ def __init__(self, data: bytes, dex_name: str = "classes.dex"):
211
+ self.data = data
212
+ self.dex_name = dex_name
213
+ if len(data) < 112 or data[:4] != b"dex\n":
214
+ raise DexError(f"{dex_name}: not a DEX file")
215
+ self.endian = _u4(data, 0x28)
216
+ if self.endian != 0x12345678:
217
+ raise DexError(f"{dex_name}: unsupported endian tag 0x{self.endian:x}")
218
+
219
+ self.string_ids_size = _u4(data, 0x38)
220
+ self.string_ids_off = _u4(data, 0x3C)
221
+ self.type_ids_size = _u4(data, 0x40)
222
+ self.type_ids_off = _u4(data, 0x44)
223
+ self.proto_ids_size = _u4(data, 0x48)
224
+ self.proto_ids_off = _u4(data, 0x4C)
225
+ self.field_ids_size = _u4(data, 0x50)
226
+ self.field_ids_off = _u4(data, 0x54)
227
+ self.method_ids_size = _u4(data, 0x58)
228
+ self.method_ids_off = _u4(data, 0x5C)
229
+ self.class_defs_size = _u4(data, 0x60)
230
+ self.class_defs_off = _u4(data, 0x64)
231
+
232
+ self._string_cache: dict[int, str] = {}
233
+ self._string_blob_size: dict[int, int] = {}
234
+ self._warnings: list[str] = []
235
+
236
+ # -- string / type helpers ------------------------------------------------
237
+
238
+ def string(self, idx: int) -> str:
239
+ cached = self._string_cache.get(idx)
240
+ if cached is not None:
241
+ return cached
242
+ if idx < 0 or idx >= self.string_ids_size:
243
+ raise DexError(f"string index {idx} out of range")
244
+ off = _u4(self.data, self.string_ids_off + 4 * idx)
245
+ _size, pos = _uleb128(self.data, off)
246
+ start = pos
247
+ while _u1(self.data, pos) != 0:
248
+ pos += 1
249
+ blob_size = (pos + 1) - off
250
+ self._string_blob_size[idx] = blob_size
251
+ value = _decode_mutf8(self.data[start:pos])
252
+ self._string_cache[idx] = value
253
+ return value
254
+
255
+ def string_blob_size(self, idx: int) -> int:
256
+ size = self._string_blob_size.get(idx)
257
+ if size is None:
258
+ self.string(idx)
259
+ size = self._string_blob_size.get(idx, 0)
260
+ return size
261
+
262
+ def type_descriptor(self, type_idx: int) -> str:
263
+ if type_idx < 0 or type_idx >= self.type_ids_size:
264
+ raise DexError(f"type index {type_idx} out of range")
265
+ descriptor_idx = _u4(self.data, self.type_ids_off + 4 * type_idx)
266
+ return self.string(descriptor_idx)
267
+
268
+ def type_string_idx(self, type_idx: int) -> int | None:
269
+ if type_idx < 0 or type_idx >= self.type_ids_size:
270
+ return None
271
+ return _u4(self.data, self.type_ids_off + 4 * type_idx)
272
+
273
+ def field_name_idx(self, field_idx: int) -> int | None:
274
+ if field_idx < 0 or field_idx >= self.field_ids_size:
275
+ return None
276
+ return _u4(self.data, self.field_ids_off + 8 * field_idx + 4)
277
+
278
+ def field_type_idx(self, field_idx: int) -> int | None:
279
+ if field_idx < 0 or field_idx >= self.field_ids_size:
280
+ return None
281
+ # field_id_item: class_idx(u2) type_idx(u2) name_idx(u4)
282
+ return _u2(self.data, self.field_ids_off + 8 * field_idx + 2)
283
+
284
+ def method_name_idx(self, method_idx: int) -> int | None:
285
+ if method_idx < 0 or method_idx >= self.method_ids_size:
286
+ return None
287
+ return _u4(self.data, self.method_ids_off + 8 * method_idx + 4)
288
+
289
+ def method_proto_idx(self, method_idx: int) -> int | None:
290
+ if method_idx < 0 or method_idx >= self.method_ids_size:
291
+ return None
292
+ # method_id_item: class_idx(u2) proto_idx(u2) name_idx(u4)
293
+ return _u2(self.data, self.method_ids_off + 8 * method_idx + 2)
294
+
295
+ def proto_string_indices(self, proto_idx: int) -> list[int]:
296
+ """String indices owned by a proto_id (shorty + return + param types)."""
297
+ if proto_idx < 0 or proto_idx >= self.proto_ids_size:
298
+ return []
299
+ base = self.proto_ids_off + 12 * proto_idx
300
+ out = [_u4(self.data, base)] # shorty_idx
301
+ out.append(self.type_string_idx(_u4(self.data, base + 4)) or -1)
302
+ params_off = _u4(self.data, base + 8)
303
+ if params_off:
304
+ count = _u4(self.data, params_off)
305
+ for i in range(min(count, 4096)):
306
+ out.append(self.type_string_idx(_u2(self.data, params_off + 4 + 2 * i)) or -1)
307
+ return [i for i in out if i >= 0]
308
+
309
+ def proto_descriptor(self, proto_idx: int) -> str:
310
+ """'(Ljava/lang/String;)I' style signature for a proto_id."""
311
+ if proto_idx < 0 or proto_idx >= self.proto_ids_size:
312
+ return "<?>"
313
+ base = self.proto_ids_off + 12 * proto_idx
314
+ try:
315
+ return_type = self.type_descriptor(_u4(self.data, base + 4))
316
+ params_off = _u4(self.data, base + 8)
317
+ params: list[str] = []
318
+ if params_off:
319
+ count = _u4(self.data, params_off)
320
+ for i in range(min(count, 4096)):
321
+ params.append(self.type_descriptor(_u2(self.data, params_off + 4 + 2 * i)))
322
+ return "(" + "".join(params) + ")" + return_type
323
+ except Exception: # noqa: BLE001
324
+ return "<?>"
325
+
326
+ def type_list_indices(self, off: int) -> tuple[list[int], int]:
327
+ """Return (type indices, byte size) for a type_list at ``off``."""
328
+ if not off:
329
+ return [], 0
330
+ count = _u4(self.data, off)
331
+ count = min(count, 4096)
332
+ indices = [_u2(self.data, off + 4 + 2 * i) for i in range(count)]
333
+ return indices, 4 + 2 * count
334
+
335
+ # -- structural size helpers ---------------------------------------------
336
+
337
+ def encoded_value_end(self, off: int, depth: int = 0) -> int:
338
+ if depth > 32:
339
+ raise DexError("encoded_value nesting too deep")
340
+ header = _u1(self.data, off)
341
+ value_type = header & 0x1F
342
+ arg = header >> 5
343
+ off += 1
344
+
345
+ # For primitive values, value_arg encodes (payload_size - 1), so the
346
+ # payload is (arg + 1) bytes after the 1-byte header. BOOLEAN is the
347
+ # exception: value_arg holds the value itself and there is no payload.
348
+ if value_type == 0x1F: # BOOLEAN: value is in value_arg, no payload
349
+ return off
350
+ if value_type in _PRIMITIVE_VALUE_TYPES:
351
+ return off + arg + 1
352
+ if value_type in _INDEX_VALUE_TYPES:
353
+ # Index-valued types (method-type / method-handle / string / type /
354
+ # field / method / enum). R8/D8 encode these with a fixed width of
355
+ # (value_arg + 1) little-endian bytes rather than uleb128; reading
356
+ # them as uleb128 desynchronises the walker on indices >= 128.
357
+ return off + arg + 1
358
+ if value_type == 0x1C: # ARRAY
359
+ count, off = _uleb128(self.data, off)
360
+ for _ in range(min(count, 65536)):
361
+ off = self.encoded_value_end(off, depth + 1)
362
+ return off
363
+ if value_type == 0x1D: # ANNOTATION
364
+ _type_idx, off = _uleb128(self.data, off)
365
+ count, off = _uleb128(self.data, off)
366
+ for _ in range(min(count, 65536)):
367
+ _name_idx, off = _uleb128(self.data, off)
368
+ off = self.encoded_value_end(off, depth + 1)
369
+ return off
370
+ if value_type == 0x1E: # NULL
371
+ return off
372
+ raise DexError(f"unknown encoded_value type 0x{value_type:x}")
373
+
374
+ def encoded_array_end(self, off: int) -> int:
375
+ count, off = _uleb128(self.data, off)
376
+ for _ in range(min(count, 65536)):
377
+ off = self.encoded_value_end(off)
378
+ return off
379
+
380
+ def annotation_item_end(self, off: int) -> int:
381
+ off += 1 # visibility byte
382
+ _type_idx, off = _uleb128(self.data, off)
383
+ count, off = _uleb128(self.data, off)
384
+ for _ in range(min(count, 65536)):
385
+ _name_idx, off = _uleb128(self.data, off)
386
+ off = self.encoded_value_end(off)
387
+ return off
388
+
389
+ def annotation_set_end(self, off: int) -> tuple[int, list[int]]:
390
+ count = _u4(self.data, off)
391
+ off += 4
392
+ items = []
393
+ for _ in range(min(count, 65536)):
394
+ items.append(_u4(self.data, off))
395
+ off += 4
396
+ return off, items
397
+
398
+ def annotation_directory_size(self, off: int) -> tuple[int, list[int]]:
399
+ """Return (size, annotation_item offsets) for an annotations_directory_item."""
400
+ class_annotations_off = _u4(self.data, off)
401
+ fields_size = _u4(self.data, off + 4)
402
+ methods_size = _u4(self.data, off + 8)
403
+ params_size = _u4(self.data, off + 12)
404
+ end = off + 16 + 8 * (fields_size + methods_size + params_size)
405
+ set_offsets: list[int] = []
406
+ if class_annotations_off:
407
+ set_offsets.append(class_annotations_off)
408
+ entry_base = off + 16
409
+ for i in range(fields_size + methods_size + params_size):
410
+ # Each entry is 8 bytes: (field/method/param) index + annotations
411
+ # offset. The set offset is the second u4, not the first.
412
+ set_offsets.append(_u4(self.data, entry_base + 8 * i + 4))
413
+ return end - off, set_offsets
414
+
415
+ def annotations_total_size(
416
+ self,
417
+ off: int,
418
+ seen_sets: set[int] | None = None,
419
+ seen_items: set[int] | None = None,
420
+ ) -> tuple[int, list[int]]:
421
+ """Total bytes of an annotation directory and everything it points to.
422
+
423
+ ``seen_sets`` / ``seen_items`` are shared across classes so that two
424
+ classes carrying the same annotation do not double-count its bytes.
425
+ """
426
+ directory_size, set_offsets = self.annotation_directory_size(off)
427
+ total = directory_size
428
+ annotation_offsets: list[int] = []
429
+ seen_sets = set() if seen_sets is None else seen_sets
430
+ seen_items = set() if seen_items is None else seen_items
431
+ for set_off in set_offsets:
432
+ if not set_off or set_off in seen_sets:
433
+ continue
434
+ seen_sets.add(set_off)
435
+ end, items = self.annotation_set_end(set_off)
436
+ total += end - set_off
437
+ for item_off in items:
438
+ if item_off not in annotation_offsets:
439
+ annotation_offsets.append(item_off)
440
+ for item_off in annotation_offsets:
441
+ if item_off in seen_items:
442
+ continue
443
+ seen_items.add(item_off)
444
+ total += self.annotation_item_end(item_off) - item_off
445
+ return total, annotation_offsets
446
+
447
+ def debug_info_end(self, off: int) -> tuple[int, list[int]]:
448
+ """Return (end offset, parameter name string indices)."""
449
+ _line_start, pos = _uleb128(self.data, off)
450
+ params, pos = _uleb128(self.data, pos)
451
+ names: list[int] = []
452
+ for _ in range(min(params, 4096)):
453
+ name_idx, pos = _uleb128(self.data, pos)
454
+ if name_idx:
455
+ names.append(name_idx - 1)
456
+ while True:
457
+ opcode = _u1(self.data, pos)
458
+ pos += 1
459
+ if opcode == 0x00: # END_SEQUENCE
460
+ return pos, names
461
+ if opcode == 0x01: # ADVANCE_PC
462
+ _v, pos = _uleb128(self.data, pos)
463
+ elif opcode == 0x02: # ADVANCE_LINE
464
+ _v, pos = _sleb128(self.data, pos)
465
+ elif opcode == 0x03: # START_LOCAL
466
+ _reg, pos = _uleb128(self.data, pos)
467
+ name_idx, pos = _uleb128(self.data, pos)
468
+ if name_idx:
469
+ names.append(name_idx - 1)
470
+ _t, pos = _uleb128(self.data, pos)
471
+ elif opcode == 0x04: # START_LOCAL_EXTENDED
472
+ _reg, pos = _uleb128(self.data, pos)
473
+ name_idx, pos = _uleb128(self.data, pos)
474
+ if name_idx:
475
+ names.append(name_idx - 1)
476
+ _t, pos = _uleb128(self.data, pos)
477
+ _s, pos = _uleb128(self.data, pos)
478
+ elif opcode in (0x05, 0x06): # END_LOCAL / RESTART_LOCAL
479
+ _reg, pos = _uleb128(self.data, pos)
480
+ elif opcode in (0x07, 0x08): # PROLOGUE_END / EPILOGUE_BEGIN
481
+ pass
482
+ elif opcode == 0x09: # SET_FILE has a uleb128 operand
483
+ _v, pos = _uleb128(self.data, pos)
484
+ # 0x0a..0xff are SPECIAL opcodes: the pc/line delta is encoded in
485
+ # the opcode byte itself, so they have NO operand. Consuming one
486
+ # here misaligns the stream and reads garbage as local names.
487
+
488
+ def code_item_size(self, off: int) -> tuple[int, int | None, int]:
489
+ """Return (total bytes, debug info offset, instruction count) for a code_item.
490
+
491
+ A code_item starts with a 16-byte header (registers, try/catch count, debug
492
+ offset, instruction count), followed by instructions and, when present, the
493
+ try/catch handler table.
494
+ """
495
+ # code_item: registers(u2) ins(u2) outs(u2) tries(u2) debug(u4) insns_size(u4)
496
+ tries_size = _u2(self.data, off + 6)
497
+ debug_off = _u4(self.data, off + 8)
498
+ insns_size = _u4(self.data, off + 12)
499
+ size = 16 + 2 * insns_size
500
+ pos = off + size
501
+ if tries_size:
502
+ if insns_size & 1: # try_item starts on a 4-byte boundary
503
+ pos += 2
504
+ pos += 8 * tries_size
505
+ handlers_size, pos = _uleb128(self.data, pos)
506
+ for _ in range(min(handlers_size, 65536)):
507
+ handled, pos = _uleb128(self.data, pos)
508
+ for _ in range(min(handled, 65536)):
509
+ _type_idx, pos = _uleb128(self.data, pos)
510
+ _addr, pos = _uleb128(self.data, pos)
511
+ # catch_all_addr exists only for a zero-length handler
512
+ if handled == 0:
513
+ _catch_all, pos = _uleb128(self.data, pos)
514
+ return pos - off, debug_off or None, insns_size
515
+
516
+ def class_data_info(
517
+ self, off: int
518
+ ) -> tuple[int, list[tuple[int, int]], list[tuple[int, int, int]]]:
519
+ """Return (byte size, [(field idx, flags)], [(method idx, flags, code_off)])."""
520
+ if not off:
521
+ return 0, [], []
522
+ static_fields, pos = _uleb128(self.data, off)
523
+ instance_fields, pos = _uleb128(self.data, pos)
524
+ direct_methods, pos = _uleb128(self.data, pos)
525
+ virtual_methods, pos = _uleb128(self.data, pos)
526
+
527
+ fields: list[tuple[int, int]] = []
528
+ methods: list[tuple[int, int, int]] = []
529
+
530
+ # Each of the four lists encodes its first index as a full value and
531
+ # the remaining ones as deltas. The delta sequence RESTARTS per list.
532
+ idx = 0
533
+ for _ in range(static_fields):
534
+ diff, pos = _uleb128(self.data, pos)
535
+ flags, pos = _uleb128(self.data, pos)
536
+ idx += diff
537
+ fields.append((idx, flags))
538
+
539
+ idx = 0
540
+ for _ in range(instance_fields):
541
+ diff, pos = _uleb128(self.data, pos)
542
+ flags, pos = _uleb128(self.data, pos)
543
+ idx += diff
544
+ fields.append((idx, flags))
545
+
546
+ idx = 0
547
+ for _ in range(direct_methods):
548
+ diff, pos = _uleb128(self.data, pos)
549
+ flags, pos = _uleb128(self.data, pos)
550
+ code_off, pos = _uleb128(self.data, pos)
551
+ idx += diff
552
+ methods.append((idx, flags, code_off))
553
+
554
+ idx = 0
555
+ for _ in range(virtual_methods):
556
+ diff, pos = _uleb128(self.data, pos)
557
+ flags, pos = _uleb128(self.data, pos)
558
+ code_off, pos = _uleb128(self.data, pos)
559
+ idx += diff
560
+ methods.append((idx, flags, code_off))
561
+ return pos - off, fields, methods
562
+
563
+ # -- main analysis --------------------------------------------------------
564
+
565
+ def analyze(self) -> DexAnalysis:
566
+ result = DexAnalysis(path=self.dex_name, file_size=len(self.data))
567
+
568
+ # Pre-compute string blob sizes (also fills the cache lazily via string()).
569
+ for i in range(self.string_ids_size):
570
+ try:
571
+ self.string(i)
572
+ except Exception as exc: # noqa: BLE001 - keep going on malformed entries
573
+ self._warnings.append(f"string[{i}]: {exc}")
574
+
575
+ # ===== Pass 1: collect raw information for every class =====
576
+ # Walk class_defs (the dex class list) and read class identity, superclass,
577
+ # interfaces, fields, methods, annotations, static values, etc. Also:
578
+ # (a) add class-owned bytes to breakdown;
579
+ # (b) build semantic parts for the change fingerprint.
580
+ records: list[dict] = []
581
+ seen_annotation_sets: set[int] = set()
582
+ seen_annotation_items: set[int] = set()
583
+ for i in range(self.class_defs_size):
584
+ base = self.class_defs_off + 32 * i
585
+ try:
586
+ class_idx = _u4(self.data, base)
587
+ access_flags = _u4(self.data, base + 4)
588
+ superclass_idx = _u4(self.data, base + 8)
589
+ interfaces_off = _u4(self.data, base + 12)
590
+ source_file_idx = _u4(self.data, base + 16)
591
+ annotations_off = _u4(self.data, base + 20)
592
+ class_data_off = _u4(self.data, base + 24)
593
+ static_values_off = _u4(self.data, base + 28)
594
+ desc = self.type_descriptor(class_idx)
595
+ except Exception as exc: # noqa: BLE001
596
+ self._warnings.append(f"class_def[{i}]: {exc}")
597
+ continue
598
+
599
+ class_string_idx = self.type_string_idx(class_idx)
600
+ record: dict = {
601
+ "descriptor": desc,
602
+ "class_idx": class_idx,
603
+ "superclass_idx": superclass_idx,
604
+ "interfaces_off": interfaces_off,
605
+ "source_file_idx": source_file_idx,
606
+ "annotations_off": annotations_off,
607
+ "class_data_off": class_data_off,
608
+ "static_values_off": static_values_off,
609
+ "access_flags": access_flags,
610
+ "strings": {class_string_idx} if class_string_idx is not None else set(),
611
+ "types": {class_idx},
612
+ "fields": set(),
613
+ "methods": set(),
614
+ "protos": set(),
615
+ "breakdown": SizeBreakdown(def_item=32),
616
+ "partial": False,
617
+ "warning": "",
618
+ "method_count": 0,
619
+ "field_count": 0,
620
+ # Human-readable, index-independent fingerprint parts. Raw DEX
621
+ # indices shift whenever a class is added/removed, so hashing
622
+ # raw bytes would mark nearly every class as "modified".
623
+ "semantic": [f"C:{access_flags}:{desc}"],
624
+ }
625
+
626
+ superclass_string_idx = self.type_string_idx(superclass_idx)
627
+ if superclass_string_idx is not None:
628
+ record["types"].add(superclass_idx)
629
+ record["strings"].add(superclass_string_idx)
630
+ try:
631
+ record["semantic"].append("S:" + self.type_descriptor(superclass_idx))
632
+ except Exception: # noqa: BLE001
633
+ pass
634
+
635
+ if interfaces_off:
636
+ try:
637
+ indices, size = self.type_list_indices(interfaces_off)
638
+ record["breakdown"].type_lists += size
639
+ try:
640
+ record["semantic"].append(
641
+ "I:" + ",".join(sorted(self.type_descriptor(t) for t in indices))
642
+ )
643
+ except Exception: # noqa: BLE001
644
+ pass
645
+ for t in indices:
646
+ record["types"].add(t)
647
+ s = self.type_string_idx(t)
648
+ if s is not None:
649
+ record["strings"].add(s)
650
+ except Exception as exc: # noqa: BLE001
651
+ record["partial"] = True
652
+ record["warning"] = f"interfaces: {exc}"
653
+
654
+ if source_file_idx < self.string_ids_size:
655
+ record["strings"].add(source_file_idx)
656
+ try:
657
+ record["semantic"].append("SF:" + self.string(source_file_idx))
658
+ except Exception: # noqa: BLE001
659
+ pass
660
+
661
+ # class_data
662
+ if class_data_off:
663
+ try:
664
+ cd_size, fields, methods = self.class_data_info(class_data_off)
665
+ record["breakdown"].class_data += cd_size
666
+ record["field_count"] = len(fields)
667
+ record["method_count"] = len(methods)
668
+ for f, flags in fields:
669
+ record["fields"].add(f)
670
+ n = self.field_name_idx(f)
671
+ name = self.string(n) if n is not None else "?"
672
+ t = self.field_type_idx(f)
673
+ type_desc = self.type_descriptor(t) if t is not None else "?"
674
+ record["semantic"].append(f"F:{flags}:{name}:{type_desc}")
675
+ if n is not None:
676
+ record["strings"].add(n)
677
+ if t is not None:
678
+ record["types"].add(t)
679
+ s = self.type_string_idx(t)
680
+ if s is not None:
681
+ record["strings"].add(s)
682
+ for m, flags, code_off in methods:
683
+ record["methods"].add(m)
684
+ n = self.method_name_idx(m)
685
+ name = self.string(n) if n is not None else "?"
686
+ if n is not None:
687
+ record["strings"].add(n)
688
+ p = self.method_proto_idx(m)
689
+ proto = self.proto_descriptor(p) if p is not None else "?"
690
+ spec = f"M:{flags}:{name}:{proto}"
691
+ if p is not None:
692
+ record["protos"].add(p)
693
+ for s in self.proto_string_indices(p):
694
+ record["strings"].add(s)
695
+ if code_off:
696
+ code_size, debug_off, insns = self.code_item_size(code_off)
697
+ record["breakdown"].code += code_size
698
+ # insns_size (instruction count) is stable; the raw
699
+ # byte size includes ULEB128 exception-handler
700
+ # indices which shift when other classes change.
701
+ spec += f":ins={insns}"
702
+ if debug_off:
703
+ try:
704
+ d_end, names = self.debug_info_end(debug_off)
705
+ record["breakdown"].debug_info += d_end - debug_off
706
+ name_values = []
707
+ for idx in names:
708
+ if idx < self.string_ids_size:
709
+ record["strings"].add(idx)
710
+ name_values.append(self.string(idx))
711
+ spec += ":dbg=" + ",".join(sorted(name_values))
712
+ except Exception as exc: # noqa: BLE001
713
+ record["partial"] = True
714
+ record["warning"] = f"debug_info: {exc}"
715
+ record["semantic"].append(spec)
716
+ except Exception as exc: # noqa: BLE001
717
+ record["partial"] = True
718
+ record["warning"] = f"class_data: {exc}"
719
+
720
+ # annotations -> also pull type indices out of the encoded values
721
+ if annotations_off:
722
+ try:
723
+ ann_size, ann_offsets = self.annotations_total_size(
724
+ annotations_off, seen_annotation_sets, seen_annotation_items
725
+ )
726
+ record["breakdown"].annotations += ann_size
727
+ for item_off in ann_offsets:
728
+ try:
729
+ self.annotation_item_end(item_off)
730
+ # Collect type_idx referenced at the head of every
731
+ # annotation_item and every encoded ANNOTATION value.
732
+ pos = item_off + 1
733
+ t, pos = _uleb128(self.data, pos)
734
+ record["types"].add(t)
735
+ s = self.type_string_idx(t)
736
+ if s is not None:
737
+ record["strings"].add(s)
738
+ try:
739
+ record["semantic"].append("A:" + self.type_descriptor(t))
740
+ except Exception: # noqa: BLE001
741
+ pass
742
+ count, pos = _uleb128(self.data, pos)
743
+ for _ in range(min(count, 4096)):
744
+ _name_idx, pos = _uleb128(self.data, pos)
745
+ pos = self._collect_value_types(pos, record, depth=0)
746
+ except Exception: # noqa: BLE001
747
+ pass
748
+ except Exception as exc: # noqa: BLE001
749
+ record["partial"] = True
750
+ record["warning"] = (record["warning"] + f"; annotations: {exc}").strip("; ")
751
+
752
+ # static values
753
+ if static_values_off:
754
+ try:
755
+ end = self.encoded_array_end(static_values_off)
756
+ record["breakdown"].static_values += end - static_values_off
757
+ pos = static_values_off
758
+ count, pos = _uleb128(self.data, pos)
759
+ for _ in range(min(count, 4096)):
760
+ pos = self._collect_value_types(pos, record, depth=0)
761
+ except Exception: # noqa: BLE001
762
+ pass
763
+
764
+ # remove invalid indices
765
+ record["strings"] = {s for s in record["strings"] if 0 <= s < self.string_ids_size}
766
+ record["types"] = {t for t in record["types"] if 0 <= t < self.type_ids_size}
767
+ record["fields"] = {f for f in record["fields"] if 0 <= f < self.field_ids_size}
768
+ record["methods"] = {m for m in record["methods"] if 0 <= m < self.method_ids_size}
769
+ record["protos"] = {p for p in record["protos"] if 0 <= p < self.proto_ids_size}
770
+ records.append(record)
771
+
772
+ # ===== Pass 2: distribute shared tables and strings to classes =====
773
+ # Pass 1 recorded which type/field/method/string entries each class refers to.
774
+ # Those tables are global; charging them per class would double-count.
775
+ # Sort classes by name (fixed order = deterministic result), then let
776
+ # the first referencing class own each shared entry (setdefault).
777
+ records.sort(key=lambda r: r["descriptor"])
778
+
779
+ owner_type: dict[int, int] = {}
780
+ owner_field: dict[int, int] = {}
781
+ owner_method: dict[int, int] = {}
782
+ owner_proto: dict[int, int] = {}
783
+ owner_string: dict[int, int] = {}
784
+
785
+ for pos_idx, rec in enumerate(records):
786
+ for t in rec["types"]:
787
+ owner_type.setdefault(t, pos_idx)
788
+ for f in rec["fields"]:
789
+ owner_field.setdefault(f, pos_idx)
790
+ for m in rec["methods"]:
791
+ owner_method.setdefault(m, pos_idx)
792
+ for p in rec["protos"]:
793
+ owner_proto.setdefault(p, pos_idx)
794
+ for s in rec["strings"]:
795
+ owner_string.setdefault(s, pos_idx)
796
+
797
+ type_counts = Counter(owner_type.values())
798
+ field_counts = Counter(owner_field.values())
799
+ method_counts = Counter(owner_method.values())
800
+ proto_counts = Counter(owner_proto.values())
801
+ string_counts = Counter(owner_string.values())
802
+ strings_by_owner: dict[int, list[int]] = {}
803
+ for string_idx, pos_idx in owner_string.items():
804
+ strings_by_owner.setdefault(pos_idx, []).append(string_idx)
805
+
806
+ for pos_idx, rec in enumerate(records):
807
+ rec["breakdown"].identifiers += (
808
+ 4 * type_counts[pos_idx]
809
+ + 8 * field_counts[pos_idx]
810
+ + 8 * method_counts[pos_idx]
811
+ + 12 * proto_counts[pos_idx]
812
+ + 4 * string_counts[pos_idx]
813
+ )
814
+ for s in strings_by_owner.get(pos_idx, []):
815
+ rec["breakdown"].strings += self.string_blob_size(s)
816
+
817
+ # ===== Semantic fingerprint: did this class really change? =====
818
+ # Only source-meaningful data enters the fingerprint: field names, method
819
+ # signatures, local variable names, annotations. Never raw dex bytes/indices.
820
+ # Adding/removing other classes renumbers every index, so raw bytes change
821
+ # even when this class's source did not; comparing them would flood the report.
822
+ digest = hashlib.sha256()
823
+ # NOTE: do NOT hash rec["strings"] here. That set is the result of
824
+ # first-owner attribution and shifts when unrelated classes change;
825
+ # hashing it re-introduces the false "modified" noise we are
826
+ # avoiding. All names that truly belong to the class are already
827
+ # in rec["semantic"] (superclass, interfaces, fields, methods,
828
+ # debug local names, annotations, source file).
829
+ for part in sorted(rec["semantic"]):
830
+ digest.update(part.encode("utf-8", "replace"))
831
+ digest.update(b"\x00")
832
+
833
+ result.classes.append(
834
+ DexClass(
835
+ name=rec["descriptor"],
836
+ dex_name=self.dex_name,
837
+ size=rec["breakdown"].total,
838
+ breakdown=rec["breakdown"],
839
+ content_hash=digest.hexdigest(),
840
+ methods=rec["method_count"],
841
+ fields=rec["field_count"],
842
+ partial=rec["partial"],
843
+ warning=rec["warning"],
844
+ semantic_parts=sorted(rec["semantic"]),
845
+ )
846
+ )
847
+
848
+ attributed = sum(c.size for c in result.classes)
849
+ result.unattributed = max(0, len(self.data) - attributed)
850
+ result.warnings = self._warnings
851
+ return result
852
+
853
+ def _collect_value_types(self, off: int, rec: dict, depth: int) -> int:
854
+ """Walk one encoded_value, recording type indices it references."""
855
+ if depth > 32:
856
+ return off
857
+ header = _u1(self.data, off)
858
+ value_type = header & 0x1F
859
+ arg = header >> 5
860
+ off += 1
861
+ if value_type == 0x1F: # BOOLEAN: value is in value_arg, no payload
862
+ return off
863
+ if value_type in _PRIMITIVE_VALUE_TYPES:
864
+ return off + arg + 1
865
+ if value_type in _INDEX_VALUE_TYPES:
866
+ # Fixed-width little-endian index (value_arg + 1 bytes), matching
867
+ # how R8/D8 emit encoded_value indices. See encoded_value_end.
868
+ width = arg + 1
869
+ val = int.from_bytes(self.data[off:off + width], "little")
870
+ off += width
871
+ if value_type == 0x18: # TYPE
872
+ rec["types"].add(val)
873
+ s = self.type_string_idx(val)
874
+ if s is not None:
875
+ rec["strings"].add(s)
876
+ elif value_type == 0x1B: # ENUM -> field
877
+ rec["fields"].add(val)
878
+ elif value_type == 0x17: # STRING
879
+ rec["strings"].add(val)
880
+ return off
881
+ if value_type == 0x1C:
882
+ count, off = _uleb128(self.data, off)
883
+ for _ in range(min(count, 4096)):
884
+ off = self._collect_value_types(off, rec, depth + 1)
885
+ return off
886
+ if value_type == 0x1D:
887
+ t, off = _uleb128(self.data, off)
888
+ rec["types"].add(t)
889
+ s = self.type_string_idx(t)
890
+ if s is not None:
891
+ rec["strings"].add(s)
892
+ count, off = _uleb128(self.data, off)
893
+ for _ in range(min(count, 4096)):
894
+ name_idx, off = _uleb128(self.data, off)
895
+ rec["strings"].add(name_idx)
896
+ off = self._collect_value_types(off, rec, depth + 1)
897
+ return off
898
+ if value_type == 0x1E:
899
+ return off
900
+ return off