split-dex 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- split_dex/__init__.py +5 -0
- split_dex/class_sets.py +135 -0
- split_dex/cli.py +227 -0
- split_dex/partition.py +229 -0
- split_dex/py.typed +0 -0
- split_dex/registry.py +214 -0
- split_dex/splitter.py +584 -0
- split_dex/strategies/__init__.py +1 -0
- split_dex/strategies/original.py +59 -0
- split_dex/strategies/package_affinity.py +40 -0
- split_dex/strategies/package_order.py +56 -0
- split_dex/strategies/single.py +23 -0
- split_dex/strategy.py +233 -0
- split_dex-0.1.0.dist-info/METADATA +253 -0
- split_dex-0.1.0.dist-info/RECORD +18 -0
- split_dex-0.1.0.dist-info/WHEEL +4 -0
- split_dex-0.1.0.dist-info/entry_points.txt +2 -0
- split_dex-0.1.0.dist-info/licenses/LICENSE +201 -0
split_dex/splitter.py
ADDED
|
@@ -0,0 +1,584 @@
|
|
|
1
|
+
"""Split the classes of an APK into grouped DEX files.
|
|
2
|
+
|
|
3
|
+
The classes are sorted into named sets (see `split_dex.class_sets`), and the strategy of each set
|
|
4
|
+
groups its classes (see `split_dex.strategy`). Each group is written as one DEX file, or as
|
|
5
|
+
several when it goes over a 64K limit of the DEX format. Every class lands in exactly one DEX
|
|
6
|
+
file, and two sets never share a file.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import collections.abc
|
|
10
|
+
import dataclasses
|
|
11
|
+
import pathlib
|
|
12
|
+
import re
|
|
13
|
+
from typing import TextIO
|
|
14
|
+
|
|
15
|
+
import dexpp
|
|
16
|
+
|
|
17
|
+
import split_dex
|
|
18
|
+
from split_dex import class_sets
|
|
19
|
+
from split_dex import partition
|
|
20
|
+
from split_dex import strategy
|
|
21
|
+
from split_dex.strategies import package_affinity
|
|
22
|
+
|
|
23
|
+
# Older DEX files are written as 035, the lowest DEX version.
|
|
24
|
+
_MIN_DEX_VERSION = 35
|
|
25
|
+
|
|
26
|
+
# A top-level DEX entry of an APK: classes.dex, then classes2.dex, classes3.dex, ... The number
|
|
27
|
+
# is read the way dexpp's analysis reads it: decimal digits that fit in 32 bits.
|
|
28
|
+
_DEX_ENTRY = re.compile(r"classes([0-9]*)\.dex")
|
|
29
|
+
_MAX_DEX_ENTRY_NUMBER = 0xFFFFFFFF
|
|
30
|
+
|
|
31
|
+
# A DEX file name that this tool writes: 0.dex, 1.dex, ... but never 07.dex.
|
|
32
|
+
_WRITTEN_DEX_FILE = re.compile(r"(0|[1-9][0-9]*)\.dex")
|
|
33
|
+
|
|
34
|
+
# The most entries a DEX file can index in each pool (see dexpp.PoolStats).
|
|
35
|
+
_MAX_TYPES = 65535
|
|
36
|
+
_MAX_PROTOS = 65535
|
|
37
|
+
_MAX_MEMBERS = 65536 # fields, methods, method handles and call sites
|
|
38
|
+
|
|
39
|
+
# How many classes an error message about a strategy's groups names.
|
|
40
|
+
_EXAMPLES = 3
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class SplitError(split_dex.Error):
|
|
44
|
+
"""Raised when the APK cannot be read or a file cannot be written."""
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def default_sets() -> tuple[class_sets.ClassSet, ...]:
|
|
48
|
+
"""The sets used when none are given: one set, `all`, grouped by `package-affinity`.
|
|
49
|
+
|
|
50
|
+
Returns:
|
|
51
|
+
The sets.
|
|
52
|
+
"""
|
|
53
|
+
return (class_sets.ClassSet("all", package_affinity.PackageAffinity()),)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclasses.dataclass(frozen=True)
|
|
57
|
+
class Options:
|
|
58
|
+
"""What to split and how.
|
|
59
|
+
|
|
60
|
+
Attributes:
|
|
61
|
+
apk_path: The input APK. Every top-level `classes*.dex` entry is read.
|
|
62
|
+
out_dir: The output directory. It is created if it does not exist.
|
|
63
|
+
sets: The sets to sort the classes into. Their DEX files are written in this order.
|
|
64
|
+
assignments: Which classes go to which set, tried in order (see
|
|
65
|
+
`split_dex.class_sets.assign`). It may be empty when there is a single set.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
apk_path: pathlib.Path
|
|
69
|
+
out_dir: pathlib.Path
|
|
70
|
+
sets: collections.abc.Sequence[class_sets.ClassSet] = dataclasses.field(
|
|
71
|
+
default_factory=default_sets
|
|
72
|
+
)
|
|
73
|
+
assignments: collections.abc.Sequence[class_sets.Assignment] = ()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@dataclasses.dataclass(frozen=True)
|
|
77
|
+
class SetResult:
|
|
78
|
+
"""What a split produced for one set.
|
|
79
|
+
|
|
80
|
+
Attributes:
|
|
81
|
+
name: The set's name.
|
|
82
|
+
classes: The number of classes in the set.
|
|
83
|
+
dex_files: The number of DEX files written for the set.
|
|
84
|
+
"""
|
|
85
|
+
|
|
86
|
+
name: str
|
|
87
|
+
classes: int
|
|
88
|
+
dex_files: int
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@dataclasses.dataclass(frozen=True)
|
|
92
|
+
class Result:
|
|
93
|
+
"""What a split produced.
|
|
94
|
+
|
|
95
|
+
Attributes:
|
|
96
|
+
total_classes: The number of classes read, each class counted once.
|
|
97
|
+
duplicate_classes: How many later copies of classes defined in more than one DEX file
|
|
98
|
+
were skipped. The first copy in multidex order is kept: the one Android loads.
|
|
99
|
+
dex_files: The number of DEX files written.
|
|
100
|
+
overflow_splits: How many groups were cut in two because they went over a 64K limit of
|
|
101
|
+
the DEX format.
|
|
102
|
+
split_blocks: The names of the blocks, such as `package Lcom/myapp/big`, that were too
|
|
103
|
+
big for one DEX file and had to be cut across files. Normally empty.
|
|
104
|
+
total_bytes: The total size of the written DEX files.
|
|
105
|
+
sets: What was produced for each set, in the order of the sets.
|
|
106
|
+
"""
|
|
107
|
+
|
|
108
|
+
total_classes: int
|
|
109
|
+
duplicate_classes: int
|
|
110
|
+
dex_files: int
|
|
111
|
+
overflow_splits: int
|
|
112
|
+
split_blocks: list[str]
|
|
113
|
+
total_bytes: int
|
|
114
|
+
sets: list[SetResult]
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def split(options: Options) -> Result:
|
|
118
|
+
"""Split the classes of an APK into grouped DEX files under `options.out_dir`.
|
|
119
|
+
|
|
120
|
+
The output directory receives:
|
|
121
|
+
|
|
122
|
+
- `<k>.dex`: the grouped DEX files, `0.dex`, `1.dex`, ..., set by set. Each one is a valid,
|
|
123
|
+
standalone DEX file.
|
|
124
|
+
- `index.csv` (`class,file`): every class descriptor and the DEX file it is in.
|
|
125
|
+
- `sets.csv` (`file,set`): the set of each DEX file.
|
|
126
|
+
|
|
127
|
+
These files replace those of an earlier run. `<k>.dex` files left by an earlier, bigger run
|
|
128
|
+
are removed, so every DEX file in the directory is listed in `index.csv`. Other files in the
|
|
129
|
+
directory are not touched.
|
|
130
|
+
|
|
131
|
+
Args:
|
|
132
|
+
options: What to split and how.
|
|
133
|
+
|
|
134
|
+
Returns:
|
|
135
|
+
The counts of what was read and written.
|
|
136
|
+
|
|
137
|
+
Raises:
|
|
138
|
+
SplitError: If the APK cannot be read or a file cannot be written.
|
|
139
|
+
split_dex.class_sets.AssignmentError: If the classes cannot be sorted into the sets.
|
|
140
|
+
split_dex.strategy.StrategyError: If a strategy returns groups that break the rules.
|
|
141
|
+
"""
|
|
142
|
+
apk = _open_apk(options.apk_path)
|
|
143
|
+
classes = _read_classes(apk, options.apk_path)
|
|
144
|
+
members = class_sets.assign(classes.descriptors, options.sets, options.assignments)
|
|
145
|
+
app = _App(apk, options.apk_path, classes)
|
|
146
|
+
plans = [_plan(class_set, members[class_set.name], app) for class_set in options.sets]
|
|
147
|
+
_create_directory(options.out_dir)
|
|
148
|
+
writer = _write_plans(options.out_dir, classes, options.sets, plans)
|
|
149
|
+
_remove_stale_dex_files(options.out_dir, writer.file_count)
|
|
150
|
+
return Result(
|
|
151
|
+
total_classes=len(classes.descriptors),
|
|
152
|
+
duplicate_classes=classes.duplicates,
|
|
153
|
+
dex_files=writer.file_count,
|
|
154
|
+
overflow_splits=writer.overflow_splits,
|
|
155
|
+
split_blocks=writer.split_blocks,
|
|
156
|
+
total_bytes=writer.total_bytes,
|
|
157
|
+
sets=[
|
|
158
|
+
SetResult(class_set.name, len(members[class_set.name]), writer.files_of(class_set.name))
|
|
159
|
+
for class_set in options.sets
|
|
160
|
+
],
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@dataclasses.dataclass(frozen=True)
|
|
165
|
+
class _ApkClasses:
|
|
166
|
+
"""The classes of an APK's DEX files. Class ids count from 0, in multidex order.
|
|
167
|
+
|
|
168
|
+
Attributes:
|
|
169
|
+
models: One model per `classes*.dex` entry, in multidex order.
|
|
170
|
+
descriptors: The descriptor of each class, indexed by class id.
|
|
171
|
+
sources: The index in `models` of the DEX file each class comes from.
|
|
172
|
+
index_of: The class id of each descriptor.
|
|
173
|
+
duplicates: How many later copies of classes were skipped.
|
|
174
|
+
version: The DEX version to write: the highest version read, and at least 035.
|
|
175
|
+
"""
|
|
176
|
+
|
|
177
|
+
models: list[dexpp.Model]
|
|
178
|
+
descriptors: list[str]
|
|
179
|
+
sources: list[int]
|
|
180
|
+
index_of: dict[str, int]
|
|
181
|
+
duplicates: int
|
|
182
|
+
version: int
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _open_apk(apk_path: pathlib.Path) -> dexpp.Apk:
|
|
186
|
+
if apk_path.exists() is False:
|
|
187
|
+
raise SplitError(f"apk not found: {apk_path}")
|
|
188
|
+
try:
|
|
189
|
+
return dexpp.Apk.open(str(apk_path))
|
|
190
|
+
except (FileNotFoundError, ValueError) as error:
|
|
191
|
+
raise SplitError(f"cannot open apk: {apk_path}") from error
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _read_classes(apk: dexpp.Apk, apk_path: pathlib.Path) -> _ApkClasses:
|
|
195
|
+
"""Read the classes of every `classes*.dex` entry, in multidex order.
|
|
196
|
+
|
|
197
|
+
A class defined in more than one DEX file is kept once: the first copy in multidex order,
|
|
198
|
+
which is the one Android loads. Keeping both would put one class in two output files.
|
|
199
|
+
"""
|
|
200
|
+
entries = _dex_entries(apk.entries)
|
|
201
|
+
if len(entries) == 0:
|
|
202
|
+
raise SplitError(f"apk has no classes*.dex: {apk_path}")
|
|
203
|
+
models: list[dexpp.Model] = []
|
|
204
|
+
descriptors: list[str] = []
|
|
205
|
+
sources: list[int] = []
|
|
206
|
+
index_of: dict[str, int] = {}
|
|
207
|
+
duplicates = 0
|
|
208
|
+
for source, entry in enumerate(entries):
|
|
209
|
+
model = _read_model(apk, entry)
|
|
210
|
+
models.append(model)
|
|
211
|
+
for descriptor in model.class_descriptors:
|
|
212
|
+
if descriptor in index_of:
|
|
213
|
+
duplicates += 1
|
|
214
|
+
continue
|
|
215
|
+
index_of[descriptor] = len(descriptors)
|
|
216
|
+
descriptors.append(descriptor)
|
|
217
|
+
sources.append(source)
|
|
218
|
+
version = max(_MIN_DEX_VERSION, *(model.version for model in models))
|
|
219
|
+
return _ApkClasses(models, descriptors, sources, index_of, duplicates, version)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _dex_entries(names: collections.abc.Iterable[str]) -> list[str]:
|
|
223
|
+
"""The top-level `classes*.dex` entry names, in multidex order.
|
|
224
|
+
|
|
225
|
+
A name listed twice in the archive is read once, as dexpp's analysis does. Ties on the number
|
|
226
|
+
(`classes2.dex` and `classes02.dex`) are broken by name.
|
|
227
|
+
"""
|
|
228
|
+
numbers: dict[str, int] = {}
|
|
229
|
+
for name in names:
|
|
230
|
+
number = _multidex_number(name)
|
|
231
|
+
if number is not None:
|
|
232
|
+
numbers.setdefault(name, number)
|
|
233
|
+
return sorted(numbers, key=lambda name: (numbers[name], name))
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def _multidex_number(name: str) -> int | None:
|
|
237
|
+
"""The multidex order of a DEX entry: 1 for classes.dex, N for classesN.dex (N >= 2)."""
|
|
238
|
+
match = _DEX_ENTRY.fullmatch(name)
|
|
239
|
+
if match is None:
|
|
240
|
+
return None
|
|
241
|
+
digits = match.group(1)
|
|
242
|
+
if digits == "":
|
|
243
|
+
return 1
|
|
244
|
+
number = int(digits)
|
|
245
|
+
if number < 2 or number > _MAX_DEX_ENTRY_NUMBER:
|
|
246
|
+
return None
|
|
247
|
+
return number
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _read_model(apk: dexpp.Apk, entry: str) -> dexpp.Model:
|
|
251
|
+
data = apk.read(entry)
|
|
252
|
+
if data is None:
|
|
253
|
+
raise SplitError(f"cannot read apk entry: {entry}")
|
|
254
|
+
try:
|
|
255
|
+
return dexpp.Model.read(data)
|
|
256
|
+
except ValueError as error:
|
|
257
|
+
raise SplitError(f"{entry}: {error}") from error
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
class _App:
|
|
261
|
+
"""What the splitter knows about the APK, shared by every set (see `strategy.App`).
|
|
262
|
+
|
|
263
|
+
The analysis of the APK's code is made on first use, by the first set that needs it.
|
|
264
|
+
"""
|
|
265
|
+
|
|
266
|
+
def __init__(self, apk: dexpp.Apk, apk_path: pathlib.Path, classes: _ApkClasses) -> None:
|
|
267
|
+
self._apk = apk
|
|
268
|
+
self._apk_path = apk_path
|
|
269
|
+
self._classes = classes
|
|
270
|
+
self._context: dexpp.AnalysisContext | None = None
|
|
271
|
+
|
|
272
|
+
@property
|
|
273
|
+
def descriptors(self) -> collections.abc.Sequence[str]:
|
|
274
|
+
"""The descriptor of every class of the APK, by class id."""
|
|
275
|
+
return self._classes.descriptors
|
|
276
|
+
|
|
277
|
+
def source_dex(self, class_id: int) -> int:
|
|
278
|
+
"""The input DEX file a class comes from: 0 for `classes.dex`, 1 for `classes2.dex`, ..."""
|
|
279
|
+
return self._classes.sources[class_id]
|
|
280
|
+
|
|
281
|
+
def links(
|
|
282
|
+
self,
|
|
283
|
+
class_ids: collections.abc.Sequence[int],
|
|
284
|
+
weights: collections.abc.Mapping[strategy.LinkKind, int],
|
|
285
|
+
) -> partition.AffinityGraph:
|
|
286
|
+
"""The links among the given classes, each link counted with the weight of its kind.
|
|
287
|
+
|
|
288
|
+
Only the code of the given classes is read, so the cost follows the size of the set.
|
|
289
|
+
"""
|
|
290
|
+
members = set(class_ids)
|
|
291
|
+
index_of = self._classes.index_of
|
|
292
|
+
graph = partition.AffinityGraph(len(self._classes.descriptors))
|
|
293
|
+
for cls in self.analysis.classes:
|
|
294
|
+
class_id = index_of.get(cls.name)
|
|
295
|
+
if class_id is None or class_id not in members:
|
|
296
|
+
continue
|
|
297
|
+
for descriptor, weight in _weighted_links(cls, weights):
|
|
298
|
+
other = index_of.get(descriptor)
|
|
299
|
+
if other is not None and other in members:
|
|
300
|
+
graph.add_edge(class_id, other, weight)
|
|
301
|
+
return graph
|
|
302
|
+
|
|
303
|
+
@property
|
|
304
|
+
def apk(self) -> dexpp.Apk:
|
|
305
|
+
"""The APK."""
|
|
306
|
+
return self._apk
|
|
307
|
+
|
|
308
|
+
@property
|
|
309
|
+
def analysis(self) -> dexpp.AnalysisContext:
|
|
310
|
+
"""The analysis of the APK's code."""
|
|
311
|
+
if self._context is None:
|
|
312
|
+
try:
|
|
313
|
+
self._context = self._apk.analysis()
|
|
314
|
+
except ValueError as error:
|
|
315
|
+
raise SplitError(f"analysis failed on apk: {self._apk_path}") from error
|
|
316
|
+
return self._context
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _weighted_links(
|
|
320
|
+
cls: dexpp.Class, weights: collections.abc.Mapping[strategy.LinkKind, int]
|
|
321
|
+
) -> collections.abc.Iterator[tuple[str, int]]:
|
|
322
|
+
"""The class descriptors a class is linked to, once per link, with the weight of the link.
|
|
323
|
+
|
|
324
|
+
Those are its superclass, the interfaces it implements, the top-level class it is nested in
|
|
325
|
+
(`LFoo$Bar;` is linked to `LFoo;`), and every class that one of its methods refers to: a
|
|
326
|
+
called method, an accessed field, or a used type (`new`, a cast, `instanceof`, ...). Links of
|
|
327
|
+
a kind that weighs 0 are skipped without reading them.
|
|
328
|
+
"""
|
|
329
|
+
superclass = cls.superclass
|
|
330
|
+
if superclass is not None and weights[strategy.LinkKind.SUPERCLASS] != 0:
|
|
331
|
+
yield superclass.descriptor, weights[strategy.LinkKind.SUPERCLASS]
|
|
332
|
+
if weights[strategy.LinkKind.INTERFACE] != 0:
|
|
333
|
+
for interface in cls.interfaces:
|
|
334
|
+
yield interface.descriptor, weights[strategy.LinkKind.INTERFACE]
|
|
335
|
+
name = cls.name
|
|
336
|
+
dollar = name.find("$")
|
|
337
|
+
if dollar != -1 and weights[strategy.LinkKind.NESTING] != 0:
|
|
338
|
+
yield name[:dollar] + ";", weights[strategy.LinkKind.NESTING]
|
|
339
|
+
yield from _code_links(cls, weights)
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _code_links(
|
|
343
|
+
cls: dexpp.Class, weights: collections.abc.Mapping[strategy.LinkKind, int]
|
|
344
|
+
) -> collections.abc.Iterator[tuple[str, int]]:
|
|
345
|
+
"""The classes the methods of a class refer to, with the weight of each reference's kind."""
|
|
346
|
+
call_weight = weights[strategy.LinkKind.CALL]
|
|
347
|
+
field_weight = weights[strategy.LinkKind.FIELD]
|
|
348
|
+
type_weight = weights[strategy.LinkKind.TYPE]
|
|
349
|
+
if call_weight == 0 and field_weight == 0 and type_weight == 0:
|
|
350
|
+
return
|
|
351
|
+
for method in cls.methods:
|
|
352
|
+
if call_weight != 0:
|
|
353
|
+
for call in method.calls:
|
|
354
|
+
yield call.target.class_descriptor, call_weight
|
|
355
|
+
if field_weight != 0:
|
|
356
|
+
for access in method.field_accesses:
|
|
357
|
+
yield access.field.class_descriptor, field_weight
|
|
358
|
+
if type_weight != 0:
|
|
359
|
+
for use in method.type_uses:
|
|
360
|
+
yield use.descriptor, type_weight
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def _plan(
|
|
364
|
+
class_set: class_sets.ClassSet, class_ids: collections.abc.Sequence[int], app: _App
|
|
365
|
+
) -> list[strategy.Group]:
|
|
366
|
+
"""Ask the set's strategy for its groups, and check them."""
|
|
367
|
+
groups = list(class_set.strategy.plan(strategy.Classes(class_ids, app)))
|
|
368
|
+
_check_groups(class_set.name, groups, class_ids, app.descriptors)
|
|
369
|
+
return groups
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _check_groups(
|
|
373
|
+
set_name: str,
|
|
374
|
+
groups: collections.abc.Sequence[strategy.Group],
|
|
375
|
+
class_ids: collections.abc.Sequence[int],
|
|
376
|
+
descriptors: collections.abc.Sequence[str],
|
|
377
|
+
) -> None:
|
|
378
|
+
"""Check that a strategy put every class of its set in exactly one non-empty block."""
|
|
379
|
+
expected = set(class_ids)
|
|
380
|
+
seen: set[int] = set()
|
|
381
|
+
for group in groups:
|
|
382
|
+
if len(group.blocks) == 0:
|
|
383
|
+
raise strategy.StrategyError(f"set {set_name}: the strategy returned an empty group")
|
|
384
|
+
for block in group.blocks:
|
|
385
|
+
if len(block.class_ids) == 0:
|
|
386
|
+
raise strategy.StrategyError(
|
|
387
|
+
f"set {set_name}: the strategy returned an empty block"
|
|
388
|
+
)
|
|
389
|
+
for class_id in block.class_ids:
|
|
390
|
+
if class_id not in expected:
|
|
391
|
+
raise strategy.StrategyError(
|
|
392
|
+
f"set {set_name}: the strategy returned class id {class_id}, which is "
|
|
393
|
+
"not in the set"
|
|
394
|
+
)
|
|
395
|
+
if class_id in seen:
|
|
396
|
+
raise strategy.StrategyError(
|
|
397
|
+
f"set {set_name}: the strategy returned {descriptors[class_id]} twice"
|
|
398
|
+
)
|
|
399
|
+
seen.add(class_id)
|
|
400
|
+
if len(seen) < len(expected):
|
|
401
|
+
missing = [descriptors[class_id] for class_id in class_ids if class_id not in seen]
|
|
402
|
+
raise strategy.StrategyError(
|
|
403
|
+
f"set {set_name}: the strategy left out {len(missing)} classes, such as "
|
|
404
|
+
f"{', '.join(missing[:_EXAMPLES])}"
|
|
405
|
+
)
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def _create_directory(out_dir: pathlib.Path) -> None:
|
|
409
|
+
try:
|
|
410
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
411
|
+
except OSError as error:
|
|
412
|
+
raise SplitError(f"cannot create output directory: {out_dir}") from error
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def _open_csv(path: pathlib.Path, header: str) -> TextIO:
|
|
416
|
+
"""Open a CSV map for writing, and write its header line."""
|
|
417
|
+
try:
|
|
418
|
+
file = path.open("w", encoding="utf-8", newline="\n")
|
|
419
|
+
except OSError as error:
|
|
420
|
+
raise SplitError(f"cannot write {path}") from error
|
|
421
|
+
file.write(f"{header}\n")
|
|
422
|
+
return file
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
class _DexWriter:
|
|
426
|
+
"""Writes groups of classes as numbered DEX files and lists them in the maps."""
|
|
427
|
+
|
|
428
|
+
def __init__(
|
|
429
|
+
self,
|
|
430
|
+
out_dir: pathlib.Path,
|
|
431
|
+
classes: _ApkClasses,
|
|
432
|
+
index_file: TextIO,
|
|
433
|
+
sets_file: TextIO,
|
|
434
|
+
) -> None:
|
|
435
|
+
self._out_dir = out_dir
|
|
436
|
+
self._classes = classes
|
|
437
|
+
self._index_file = index_file
|
|
438
|
+
self._sets_file = sets_file
|
|
439
|
+
self._set_file_counts: dict[str, int] = {}
|
|
440
|
+
self._total_bytes = 0
|
|
441
|
+
self._overflow_splits = 0
|
|
442
|
+
self._split_blocks: set[str] = set()
|
|
443
|
+
|
|
444
|
+
@property
|
|
445
|
+
def file_count(self) -> int:
|
|
446
|
+
"""The number of DEX files written."""
|
|
447
|
+
return sum(self._set_file_counts.values())
|
|
448
|
+
|
|
449
|
+
@property
|
|
450
|
+
def total_bytes(self) -> int:
|
|
451
|
+
"""The total size of the DEX files written."""
|
|
452
|
+
return self._total_bytes
|
|
453
|
+
|
|
454
|
+
@property
|
|
455
|
+
def overflow_splits(self) -> int:
|
|
456
|
+
"""How many groups were cut in two because they went over a 64K limit."""
|
|
457
|
+
return self._overflow_splits
|
|
458
|
+
|
|
459
|
+
@property
|
|
460
|
+
def split_blocks(self) -> list[str]:
|
|
461
|
+
"""The names of the blocks that had to be cut across files, sorted."""
|
|
462
|
+
return sorted(self._split_blocks)
|
|
463
|
+
|
|
464
|
+
def files_of(self, set_name: str) -> int:
|
|
465
|
+
"""The number of DEX files written for a set."""
|
|
466
|
+
return self._set_file_counts.get(set_name, 0)
|
|
467
|
+
|
|
468
|
+
def write_group(self, group: strategy.Group, set_name: str) -> None:
|
|
469
|
+
"""Write a group as one DEX file, or as several if it goes over a 64K limit.
|
|
470
|
+
|
|
471
|
+
A group that goes over a limit is cut in two (see `_cut`), and each part is written
|
|
472
|
+
again, the first part first.
|
|
473
|
+
"""
|
|
474
|
+
pending = [list(group.blocks)]
|
|
475
|
+
while len(pending) > 0:
|
|
476
|
+
blocks = pending.pop()
|
|
477
|
+
class_ids = [class_id for block in blocks for class_id in block.class_ids]
|
|
478
|
+
data = self._encode(class_ids)
|
|
479
|
+
if data is None:
|
|
480
|
+
self._overflow_splits += 1
|
|
481
|
+
first, second = self._cut(blocks)
|
|
482
|
+
pending.append(second)
|
|
483
|
+
pending.append(first)
|
|
484
|
+
else:
|
|
485
|
+
self._write_file(class_ids, set_name, data)
|
|
486
|
+
|
|
487
|
+
def _encode(self, class_ids: list[int]) -> bytes | None:
|
|
488
|
+
"""The classes as DEX bytes, or None if they go over a 64K limit and can be cut."""
|
|
489
|
+
model = dexpp.Model(self._classes.version)
|
|
490
|
+
for class_id in class_ids:
|
|
491
|
+
source = self._classes.models[self._classes.sources[class_id]]
|
|
492
|
+
model.copy_class(source, self._classes.descriptors[class_id])
|
|
493
|
+
try:
|
|
494
|
+
return model.write()
|
|
495
|
+
except ValueError as error:
|
|
496
|
+
if len(class_ids) > 1 and _overflows(model.pool_stats) is True:
|
|
497
|
+
return None
|
|
498
|
+
raise SplitError(f"write failed: {error}") from error
|
|
499
|
+
|
|
500
|
+
def _cut(
|
|
501
|
+
self, blocks: list[strategy.Block]
|
|
502
|
+
) -> tuple[list[strategy.Block], list[strategy.Block]]:
|
|
503
|
+
"""Cut blocks that go over a 64K limit in two.
|
|
504
|
+
|
|
505
|
+
The cut falls at the block boundary nearest the middle (the first one on a tie). A
|
|
506
|
+
single block is cut in its middle, and its name is recorded.
|
|
507
|
+
"""
|
|
508
|
+
half = sum(len(block.class_ids) for block in blocks) // 2
|
|
509
|
+
if len(blocks) == 1:
|
|
510
|
+
block = blocks[0]
|
|
511
|
+
if block.name is not None:
|
|
512
|
+
self._split_blocks.add(block.name)
|
|
513
|
+
return (
|
|
514
|
+
[strategy.Block(block.class_ids[:half], block.name)],
|
|
515
|
+
[strategy.Block(block.class_ids[half:], block.name)],
|
|
516
|
+
)
|
|
517
|
+
boundaries: list[int] = []
|
|
518
|
+
position = 0
|
|
519
|
+
for block in blocks[:-1]:
|
|
520
|
+
position += len(block.class_ids)
|
|
521
|
+
boundaries.append(position)
|
|
522
|
+
cut = min(range(len(boundaries)), key=lambda index: abs(boundaries[index] - half))
|
|
523
|
+
return blocks[: cut + 1], blocks[cut + 1 :]
|
|
524
|
+
|
|
525
|
+
def _write_file(self, class_ids: list[int], set_name: str, data: bytes) -> None:
|
|
526
|
+
name = f"{self.file_count}.dex"
|
|
527
|
+
path = self._out_dir / name
|
|
528
|
+
try:
|
|
529
|
+
path.write_bytes(data)
|
|
530
|
+
except OSError as error:
|
|
531
|
+
raise SplitError(f"cannot write DEX: {path}") from error
|
|
532
|
+
for class_id in class_ids:
|
|
533
|
+
self._index_file.write(f"{self._classes.descriptors[class_id]},{name}\n")
|
|
534
|
+
self._sets_file.write(f"{name},{set_name}\n")
|
|
535
|
+
self._total_bytes += len(data)
|
|
536
|
+
self._set_file_counts[set_name] = self.files_of(set_name) + 1
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def _overflows(stats: dexpp.PoolStats) -> bool:
|
|
540
|
+
"""Whether a pool has more entries than a DEX file can index."""
|
|
541
|
+
return (
|
|
542
|
+
stats.types > _MAX_TYPES
|
|
543
|
+
or stats.protos > _MAX_PROTOS
|
|
544
|
+
or max(stats.fields, stats.methods, stats.method_handles, stats.call_sites) > _MAX_MEMBERS
|
|
545
|
+
)
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def _write_plans(
|
|
549
|
+
out_dir: pathlib.Path,
|
|
550
|
+
classes: _ApkClasses,
|
|
551
|
+
sets: collections.abc.Sequence[class_sets.ClassSet],
|
|
552
|
+
plans: collections.abc.Sequence[collections.abc.Sequence[strategy.Group]],
|
|
553
|
+
) -> _DexWriter:
|
|
554
|
+
"""Write the groups of every set as DEX files, set by set, and list them in the maps."""
|
|
555
|
+
with (
|
|
556
|
+
_open_csv(out_dir / "index.csv", "class,file") as index_file,
|
|
557
|
+
_open_csv(out_dir / "sets.csv", "file,set") as sets_file,
|
|
558
|
+
):
|
|
559
|
+
writer = _DexWriter(out_dir, classes, index_file, sets_file)
|
|
560
|
+
for class_set, groups in zip(sets, plans, strict=True):
|
|
561
|
+
for group in groups:
|
|
562
|
+
writer.write_group(group, class_set.name)
|
|
563
|
+
return writer
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def _remove_stale_dex_files(out_dir: pathlib.Path, file_count: int) -> None:
|
|
567
|
+
"""Remove the `<k>.dex` files past the last one written, left by an earlier, bigger run.
|
|
568
|
+
|
|
569
|
+
Afterwards every DEX file in the directory is listed in `index.csv`. Only names this tool
|
|
570
|
+
writes are touched.
|
|
571
|
+
"""
|
|
572
|
+
stale = [path for path in out_dir.iterdir() if _is_stale_dex_file(path, file_count) is True]
|
|
573
|
+
for path in stale:
|
|
574
|
+
try:
|
|
575
|
+
path.unlink()
|
|
576
|
+
except OSError as error:
|
|
577
|
+
raise SplitError(f"cannot remove stale DEX: {path}") from error
|
|
578
|
+
|
|
579
|
+
|
|
580
|
+
def _is_stale_dex_file(path: pathlib.Path, file_count: int) -> bool:
|
|
581
|
+
match = _WRITTEN_DEX_FILE.fullmatch(path.name)
|
|
582
|
+
if match is None or path.is_dir() is True:
|
|
583
|
+
return False
|
|
584
|
+
return int(match.group(1)) >= file_count
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""The strategies built into split-dex. `split_dex.registry` lists them by name."""
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""The `original` strategy: keep the classes in the DEX files they came from."""
|
|
2
|
+
|
|
3
|
+
import collections.abc
|
|
4
|
+
import heapq
|
|
5
|
+
|
|
6
|
+
from split_dex import strategy
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Original:
|
|
10
|
+
"""Keep the classes in the DEX files they came from: one file per input DEX file."""
|
|
11
|
+
|
|
12
|
+
def __init__(self, *, max_files: int | None = None) -> None:
|
|
13
|
+
"""Create the strategy.
|
|
14
|
+
|
|
15
|
+
Args:
|
|
16
|
+
max_files: The most DEX files to write. When the classes come from more input DEX
|
|
17
|
+
files, the smallest groups are merged. None keeps one file per input DEX file.
|
|
18
|
+
|
|
19
|
+
Raises:
|
|
20
|
+
ValueError: If `max_files` is below 1.
|
|
21
|
+
"""
|
|
22
|
+
if max_files is not None and max_files < 1:
|
|
23
|
+
raise ValueError("max_files must be >= 1")
|
|
24
|
+
self._max_files = max_files
|
|
25
|
+
|
|
26
|
+
def plan(self, classes: strategy.Classes) -> list[strategy.Group]:
|
|
27
|
+
"""Group the classes by the input DEX file they come from, in multidex order.
|
|
28
|
+
|
|
29
|
+
Each class is a block of its own, so a group that goes over a 64K limit is cut in the
|
|
30
|
+
middle.
|
|
31
|
+
|
|
32
|
+
Args:
|
|
33
|
+
classes: The classes to group.
|
|
34
|
+
|
|
35
|
+
Returns:
|
|
36
|
+
The groups.
|
|
37
|
+
"""
|
|
38
|
+
by_source: dict[int, list[int]] = {}
|
|
39
|
+
for class_id in classes.ids:
|
|
40
|
+
by_source.setdefault(classes.source_dex(class_id), []).append(class_id)
|
|
41
|
+
groups = [by_source[source] for source in sorted(by_source)]
|
|
42
|
+
if self._max_files is not None and len(groups) > self._max_files:
|
|
43
|
+
groups = _merge(groups, self._max_files)
|
|
44
|
+
return [strategy.Group.of_classes(group) for group in groups]
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _merge(groups: collections.abc.Sequence[list[int]], count: int) -> list[list[int]]:
|
|
48
|
+
"""Merge groups into `count` bins: the biggest group first, each into the smallest bin.
|
|
49
|
+
|
|
50
|
+
Ties keep the order of the groups and take the lowest bin.
|
|
51
|
+
"""
|
|
52
|
+
bins: list[list[int]] = [[] for _ in range(count)]
|
|
53
|
+
# A heap of (class count, bin index). In ascending order, the list is already a heap.
|
|
54
|
+
sizes = [(0, index) for index in range(count)]
|
|
55
|
+
for group in sorted(groups, key=len, reverse=True):
|
|
56
|
+
size, index = heapq.heappop(sizes)
|
|
57
|
+
bins[index].extend(group)
|
|
58
|
+
heapq.heappush(sizes, (size + len(group), index))
|
|
59
|
+
return bins
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
"""The `package-affinity` strategy: group classes by how they are linked, whole packages."""
|
|
2
|
+
|
|
3
|
+
from split_dex import partition
|
|
4
|
+
from split_dex import strategy
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class PackageAffinity:
|
|
8
|
+
"""Group classes by how they are linked, each package whole, about `cap` classes per file."""
|
|
9
|
+
|
|
10
|
+
def __init__(self, *, cap: int = 1000) -> None:
|
|
11
|
+
"""Create the strategy.
|
|
12
|
+
|
|
13
|
+
Args:
|
|
14
|
+
cap: The target number of classes per DEX file.
|
|
15
|
+
|
|
16
|
+
Raises:
|
|
17
|
+
ValueError: If `cap` is below 1.
|
|
18
|
+
"""
|
|
19
|
+
if cap < 1:
|
|
20
|
+
raise ValueError("cap must be >= 1")
|
|
21
|
+
self._cap = cap
|
|
22
|
+
|
|
23
|
+
def plan(self, classes: strategy.Classes) -> list[strategy.Group]:
|
|
24
|
+
"""Group the classes, as `split_dex.partition.partition_by_package_affinity` does.
|
|
25
|
+
|
|
26
|
+
Each package is a block named after it, so a group that goes over a 64K limit is cut at
|
|
27
|
+
a package boundary.
|
|
28
|
+
|
|
29
|
+
Args:
|
|
30
|
+
classes: The classes to group.
|
|
31
|
+
|
|
32
|
+
Returns:
|
|
33
|
+
The groups.
|
|
34
|
+
"""
|
|
35
|
+
if len(classes.ids) == 0:
|
|
36
|
+
return []
|
|
37
|
+
groups = partition.partition_by_package_affinity(
|
|
38
|
+
classes.ids, classes.links(), classes.descriptors, self._cap
|
|
39
|
+
)
|
|
40
|
+
return [strategy.Group.of_packages(group, classes.descriptors) for group in groups]
|