split-dex 0.1.0__py3-none-any.whl

split_dex/strategies/package_order.py

@@ -0,0 +1,56 @@
+"""The `package-order` strategy: whole packages in name order, without code analysis."""
+
+import itertools
+
+from split_dex import partition
+from split_dex import strategy
+
+
+class PackageOrder:
+    """Put whole packages, sorted by name, into files of about `cap` classes; no code analysis."""
+
+    def __init__(self, *, cap: int = 1000) -> None:
+        """Create the strategy.
+
+        Args:
+            cap: The target number of classes per DEX file.
+
+        Raises:
+            ValueError: If `cap` is below 1.
+        """
+        if cap < 1:
+            raise ValueError("cap must be >= 1")
+        self._cap = cap
+
+    def plan(self, classes: strategy.Classes) -> list[strategy.Group]:
+        """Sort the classes by package, then by name, and fill files with whole packages.
+
+        The classes are sorted by package first: sorted by descriptor alone, a package would be
+        interleaved with its subpackages (`Lcom/a/z;` sorts after `Lcom/a/m/X;`). Packages next
+        to each other in name order are usually parts of one library or feature, so this keeps
+        related code together without reading any code: it is fast, and a new version of the app
+        moves few classes between files. Each package is a block named after it, so a group that
+        goes over a 64K limit is cut at a package boundary.
+
+        Args:
+            classes: The classes to group.
+
+        Returns:
+            The groups.
+        """
+        descriptors = classes.descriptors
+        ordered = sorted(
+            classes.ids,
+            key=lambda class_id: (
+                partition.package_of(descriptors[class_id]),
+                descriptors[class_id],
+            ),
+        )
+        runs = itertools.groupby(
+            ordered, key=lambda class_id: partition.package_of(descriptors[class_id])
+        )
+        packages = [list(ids) for _, ids in runs]
+        return [
+            strategy.Group.of_packages(group, descriptors)
+            for group in partition.fill_groups(packages, self._cap)
+        ]

split_dex/strategies/single.py

@@ -0,0 +1,23 @@
+"""The `single` strategy: all the classes in one DEX file."""
+
+from split_dex import strategy
+
+
+class Single:
+    """Put all the classes in one DEX file, cut only where it goes over a 64K limit."""
+
+    def plan(self, classes: strategy.Classes) -> list[strategy.Group]:
+        """Put every class in one group, in multidex order.
+
+        Each class is a block of its own, so a group that goes over a 64K limit is cut in the
+        middle, and the halves are cut again until each one fits.
+
+        Args:
+            classes: The classes to group.
+
+        Returns:
+            One group, or none when the set is empty.
+        """
+        if len(classes.ids) == 0:
+            return []
+        return [strategy.Group.of_classes(classes.ids)]

split_dex/strategy.py

@@ -0,0 +1,233 @@
+"""The interface between the splitter and the strategies that group classes into DEX files.
+
+The splitter sorts the classes of an APK into sets (see `split_dex.class_sets`). For each set, a
+strategy decides how its classes are grouped: it returns groups, and each group becomes one DEX
+file, or several when it goes over a 64K limit of the DEX format.
+
+A strategy is a class whose keyword-only constructor parameters are its settings and whose
+`plan` method returns the groups. Built-in strategies are registered in `split_dex.registry`; a
+user's strategy is given on the command line as `FILE.py:CLASS` or `MODULE:CLASS`, or installed
+as a plugin (see `split_dex.registry`).
+"""
+
+import collections.abc
+import dataclasses
+import enum
+import itertools
+import typing
+from typing import Self
+
+import dexpp
+
+import split_dex
+from split_dex import partition
+
+
+class StrategyError(split_dex.Error):
+    """Raised when a strategy cannot be created, or returns groups that break the rules."""
+
+
+@enum.unique
+class LinkKind(enum.Enum):
+    """The kinds of links between two classes."""
+
+    SUPERCLASS = "superclass"
+    """One class is the superclass of the other."""
+    INTERFACE = "interface"
+    """One class implements the other, an interface."""
+    NESTING = "nesting"
+    """One class is nested in the other: `LFoo$Bar;` is linked to `LFoo;`."""
+    CALL = "call"
+    """A method of one class calls a method of the other."""
+    FIELD = "field"
+    """A method of one class reads or writes a field of the other."""
+    TYPE = "type"
+    """A method of one class uses the other as a type: `new`, a cast, `instanceof`, ..."""
+
+
+@dataclasses.dataclass(frozen=True)
+class Block:
+    """Classes that should stay in the same DEX file.
+
+    When a group goes over a 64K limit, the splitter cuts it at the block boundary nearest its
+    middle. Only a group of a single block is cut inside that block.
+
+    Attributes:
+        class_ids: The classes, in write order.
+        name: How the warning names the block when it has to be cut, such as
+            `package Lcom/a`. None for a block that can be cut without a warning.
+    """
+
+    class_ids: tuple[int, ...]
+    name: str | None = None
+
+
+@dataclasses.dataclass(frozen=True)
+class Group:
+    """The classes of one DEX file, in write order, split into blocks.
+
+    Attributes:
+        blocks: The blocks, in write order.
+    """
+
+    blocks: tuple[Block, ...]
+
+    @classmethod
+    def of_classes(cls, class_ids: collections.abc.Iterable[int]) -> Self:
+        """A group with one block per class, so that a cut can fall anywhere.
+
+        Args:
+            class_ids: The classes, in write order.
+
+        Returns:
+            The group.
+        """
+        return cls(tuple(Block((class_id,)) for class_id in class_ids))
+
+    @classmethod
+    def of_packages(
+        cls, class_ids: collections.abc.Iterable[int], descriptors: collections.abc.Sequence[str]
+    ) -> Self:
+        """A group with one block per package, named after it.
+
+        A group that goes over a 64K limit is then cut at a package boundary, and a package that
+        has to be cut is named in the warning.
+
+        Args:
+            class_ids: The classes, in write order. The classes of a package should be next to
+                each other: each run of one package's classes becomes a block.
+            descriptors: The descriptor of every class, by class id.
+
+        Returns:
+            The group.
+        """
+        runs = itertools.groupby(
+            class_ids, key=lambda class_id: partition.package_of(descriptors[class_id])
+        )
+        return cls(tuple(Block(tuple(ids), f"package {package}") for package, ids in runs))
+
+    @property
+    def class_ids(self) -> list[int]:
+        """Every class of the group, in write order."""
+        return [class_id for block in self.blocks for class_id in block.class_ids]
+
+
+class App(typing.Protocol):
+    """What the splitter knows about the APK. Every set shares it."""
+
+    @property
+    def descriptors(self) -> collections.abc.Sequence[str]:
+        """The descriptor of every class of the APK, by class id."""
+        ...
+
+    def source_dex(self, class_id: int) -> int:
+        """The input DEX file a class comes from: 0 for `classes.dex`, 1 for `classes2.dex`, ..."""
+        ...
+
+    def links(
+        self,
+        class_ids: collections.abc.Sequence[int],
+        weights: collections.abc.Mapping[LinkKind, int],
+    ) -> partition.AffinityGraph:
+        """The links among the given classes, each link counted with the weight of its kind."""
+        ...
+
+    @property
+    def apk(self) -> dexpp.Apk:
+        """The APK."""
+        ...
+
+    @property
+    def analysis(self) -> dexpp.AnalysisContext:
+        """The analysis of the APK's code."""
+        ...
+
+
+class Classes:
+    """A set of classes to group, and what a strategy can know about them.
+
+    Class ids count every class of the APK, in multidex order, so they index `descriptors`
+    directly. A class defined in more than one DEX file is counted once, for its first copy.
+    """
+
+    def __init__(self, ids: collections.abc.Sequence[int], app: App) -> None:
+        """Create the view of a set of classes.
+
+        Args:
+            ids: The classes of the set, in multidex order.
+            app: What the splitter knows about the APK.
+        """
+        self._ids = ids
+        self._app = app
+
+    @property
+    def ids(self) -> collections.abc.Sequence[int]:
+        """The classes of the set, in multidex order."""
+        return self._ids
+
+    @property
+    def descriptors(self) -> collections.abc.Sequence[str]:
+        """The descriptor of every class of the APK, by class id."""
+        return self._app.descriptors
+
+    def source_dex(self, class_id: int) -> int:
+        """The input DEX file a class comes from: 0 for `classes.dex`, 1 for `classes2.dex`, ...
+
+        Args:
+            class_id: The class.
+
+        Returns:
+            The position of its DEX file in multidex order.
+        """
+        return self._app.source_dex(class_id)
+
+    def links(
+        self, weights: collections.abc.Mapping[LinkKind, int] | None = None
+    ) -> partition.AffinityGraph:
+        """The links among the classes of the set, counted per pair of classes.
+
+        The graph is built on each call, from the code of the classes of the set.
+
+        Args:
+            weights: The weight of each kind of link. A kind left out weighs 1, and a kind that
+                weighs 0 is ignored. None weighs every link 1.
+
+        Returns:
+            The graph, over class ids. The weight of an edge is the sum of the weights of the
+            links between its two classes, in both directions.
+        """
+        merged = dict.fromkeys(LinkKind, 1)
+        if weights is not None:
+            merged.update(weights)
+        return self._app.links(self._ids, merged)
+
+    @property
+    def apk(self) -> dexpp.Apk:
+        """The APK, for its manifest, resources or entries."""
+        return self._app.apk
+
+    @property
+    def analysis(self) -> dexpp.AnalysisContext:
+        """The analysis of the APK's code: classes, methods, call graph, class hierarchy, xrefs.
+
+        It covers every class of the APK, not only the set. It is made on first use and shared by
+        every set.
+        """
+        return self._app.analysis
+
+
+@typing.runtime_checkable
+class Strategy(typing.Protocol):
+    """How the classes of a set are grouped into DEX files."""
+
+    def plan(self, classes: Classes) -> collections.abc.Sequence[Group]:
+        """Group the classes of a set.
+
+        Args:
+            classes: The classes to group.
+
+        Returns:
+            The groups, in the order their DEX files are written. Every class of the set must be
+            in exactly one group, and no group or block may be empty.
+        """
+        ...

split_dex-0.1.0.dist-info/METADATA

@@ -0,0 +1,253 @@
+Metadata-Version: 2.5
+Name: split-dex
+Version: 0.1.0
+Summary: Split the classes of an Android APK into grouped DEX files
+Project-URL: Homepage, https://github.com/burogurama/split-dex
+Project-URL: Issues, https://github.com/burogurama/split-dex/issues
+Project-URL: Changelog, https://github.com/burogurama/split-dex/blob/main/CHANGELOG.md
+Author: Mohamed Nasser
+License-Expression: Apache-2.0
+License-File: LICENSE
+Keywords: android,apk,dex,multidex,split
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Software Development :: Build Tools
+Classifier: Typing :: Typed
+Requires-Python: >=3.14
+Requires-Dist: dexpp<0.3,>=0.2.0
+Description-Content-Type: text/markdown
+
+# split-dex
+
+`split-dex` splits the classes of an Android APK into grouped DEX files.
+
+It reads every `classes*.dex` in the APK, sorts the classes into sets, and
+splits each set into DEX files with a strategy. Both are chosen on the command
+line. Every class ends up in exactly one output DEX file, two sets never share a
+file, and maps record where each class went.
+
+It is written in Python, on the Python binding of
+[dexpp](https://github.com/burogurama/dexpp), which reads, analyses and writes
+the DEX files.
+
+## Usage
+
+```sh
+split-dex --apk app.apk --out out/
+```
+
+With no other option, every class is in one set, `all`, split by the
+`package-affinity` strategy into files of about 1000 classes.
+
+| Option | Meaning |
+| ------ | ------- |
+| `--apk FILE` | Input APK. All its `classes*.dex` entries are read. Required. |
+| `--out DIR` | Output directory. Created if it does not exist. Required. |
+| `--set NAME=STRATEGY[,KEY=VALUE...]` | A set of classes and the strategy that splits it, with the strategy's settings. `STRATEGY` is a built-in or plugin name (`--list-strategies` shows them), or your own class as `MODULE:CLASS` or `FILE.py:CLASS` (see [Writing a strategy](#writing-a-strategy)). Repeat it for more sets. The DEX files are written set by set, in this order. |
+| `--assign PATTERN[,PATTERN...]=NAME` | Send the classes whose names match a pattern to the set `NAME`. Repeat it; each class goes to the first pattern it matches. |
+| `--list-strategies` | Show the strategies and their settings. |
+
+With a single `--set`, `--assign` is not needed: every class goes to that set.
+With several, every class must match some pattern; end with `'**=NAME'` to
+catch the rest.
+
+A DEX file can reference at most 65,536 methods, 65,536 fields and 65,535
+types. When a group of classes goes over one of these limits, it is cut in two,
+and each half is written again (and cut again if needed). The cut falls nearest
+the middle, between the units the strategy keeps together, such as packages.
+Only a unit too big for one DEX file is cut inside, and `split-dex` prints a
+warning that names it.
+
+If the APK defines a class in more than one DEX file, only the first copy in
+multidex order is kept. That is the copy Android loads.
+
+### Patterns
+
+Patterns match class names such as `com.myapp.Main` or `com.myapp.Outer$Inner`.
+`*` matches any characters except a dot, and `**` matches any characters:
+
+| Pattern | Matches |
+| ------- | ------- |
+| `com.myapp.**` | Every class under `com.myapp`, subpackages included. |
+| `com.myapp.*` | The classes directly in `com.myapp`. |
+| `com.myapp.Main` | One class. |
+| `**` | Every class. |
+| `@FILE` | The patterns listed in `FILE`, one per line. Blank lines and lines starting with `#` are skipped. |
+
+### Examples
+
+Smaller files:
+
+```sh
+split-dex --apk app.apk --out out/ --set all=package-affinity,cap=500
+```
+
+Split the app's own classes, and put all the library classes in one DEX file
+(cut in halves where it goes over the 64K limits):
+
+```sh
+split-dex --apk app.apk --out out/ \
+    --set app=package-affinity,cap=500 \
+    --set libraries=single \
+    --assign 'com.myapp.**=app' \
+    --assign '**=libraries'
+```
+
+Split the app's own classes, but keep some of its packages with the libraries,
+which stay in their original DEX files (at most 16):
+
+```sh
+split-dex --apk app.apk --out out/ \
+    --set app=package-affinity,cap=1000 \
+    --set libraries=original,max_files=16 \
+    --assign '@startup-packages.txt=libraries' \
+    --assign 'com.myapp.**=app' \
+    --assign '**=libraries'
+```
+
+## Output
+
+In the output directory:
+
+| File        | Content                                                 |
+| ----------- | ------------------------------------------------------- |
+| `<k>.dex`   | The DEX files: `0.dex`, `1.dex`, ..., set by set.       |
+| `index.csv` | `class,file`: every class descriptor and its DEX file.  |
+| `sets.csv`  | `file,set`: the set of each DEX file.                   |
+
+Each DEX file is a valid, standalone DEX.
+
+Running again into the same directory replaces these files. If an earlier run
+wrote more `<k>.dex` files, the extra ones are removed, so every DEX file in the
+directory is listed in `index.csv`. Other files in the directory are not
+touched.
+
+At the end, `split-dex` prints a summary: how many classes were read, how many
+files were written, how many groups had to be cut for the 64K limits, and the
+classes and files of each set.
+
+## Install
+
+split-dex needs Python 3.14 or newer. Install the `split-dex` command from
+PyPI with [uv](https://docs.astral.sh/uv/) or pip:
+
+```sh
+uv tool install split-dex
+# or
+pip install split-dex
+```
+
+Its dependency dexpp has prebuilt wheels for Linux x86_64 (glibc 2.28+ and
+musl). On other platforms, the installer builds dexpp from source, which needs
+a C++23 compiler (GCC 13 or newer, or Clang 19 or newer) and CMake 3.18 or
+newer.
+
+From a checkout, `uv tool install .` installs the command.
+
+## Writing a strategy
+
+The command-line options cover the common setups. A strategy of your own is for
+the cases they don't. A strategy is a Python class:
+
+```python
+from split_dex import strategy
+
+
+class OnePerFile:
+    """Each class in a DEX file of its own, up to `limit` files."""
+
+    def __init__(self, *, limit: int = 100) -> None:
+        self._limit = limit
+
+    def plan(self, classes: strategy.Classes) -> list[strategy.Group]:
+        ids = list(classes.ids)
+        groups = [strategy.Group.of_classes([class_id]) for class_id in ids[: self._limit - 1]]
+        return groups + [strategy.Group.of_classes(ids[self._limit - 1 :])]
+```
+
+- Its keyword-only constructor parameters are its settings. The command line
+  converts each `KEY=VALUE` to the parameter's type: int, float, bool, str or
+  Path, optionally `| None`.
+- `plan(classes)` receives the classes of one set and returns the groups to
+  write, in order. Each group (`strategy.Group`) becomes one DEX file, and is
+  made of blocks (`strategy.Block`): classes that should stay in one file.
+  When a group goes over a 64K limit, the splitter cuts it at the block
+  boundary nearest its middle. `Group.of_classes(ids)` makes one block per
+  class.
+- The splitter checks that every class of the set is in exactly one group;
+  otherwise the run fails with an error that names the strategy's mistake.
+
+What `classes` (`strategy.Classes`) gives:
+
+| Member | Meaning |
+| ------ | ------- |
+| `ids` | The class ids of the set, in multidex order. |
+| `descriptors` | The descriptor of every class of the APK, indexed by class id. |
+| `source_dex(id)` | The input DEX file of a class: 0 for `classes.dex`, 1 for `classes2.dex`, ... |
+| `links(weights=None)` | The links among the classes of the set, as a `partition.AffinityGraph`. `weights` maps a `strategy.LinkKind` (superclass, interface, nesting, call, field, type) to its weight; a kind left out weighs 1, and 0 ignores it. |
+| `analysis` | dexpp's `AnalysisContext` for the whole APK: classes, methods, call graph, class hierarchy, cross-references. |
+| `apk` | dexpp's `Apk`: manifest, resources, entries. |
+
+There are four ways to use it:
+
+- From a file: `--set all=./my_strategy.py:OnePerFile,limit=8`.
+- From a module Python can import: `--set all=my_package.strategies:OnePerFile`.
+- As a plugin. An installed package declares the strategy under the
+  entry-point group `split_dex.strategies` in its `pyproject.toml`; it is then
+  used by name, and `--list-strategies` shows it:
+
+  ```toml
+  [project.entry-points."split_dex.strategies"]
+  one-per-file = "my_package.strategies:OnePerFile"
+  ```
+
+- Built into split-dex: put the module in `split_dex/strategies/` and add the
+  class to `STRATEGIES` in `split_dex/registry.py`.
+
+A strategy is ordinary Python code that runs inside split-dex, so only use
+files, modules and plugins you trust.
+
+From Python, `split_dex.splitter.split()` takes `split_dex.splitter.Options`,
+whose sets can use any object with a `plan` method.
+
+## Development
+
+```sh
+uv sync                     # .venv with split-dex, dexpp and the dev tools
+uv run pytest               # the tests
+uv run ruff format --check  # formatting
+uv run ruff check           # lint
+uv run mypy                 # types
+```
+
+The tests build small APKs in memory. Their classes are copies of the template
+classes in `tests/data/templates.dex`, renamed for each test. The templates are
+compiled from `tests/data/Templates.java`; `tests/data/make_templates.sh`
+rebuilds them (it needs `javac` and the Android SDK's `d8`).
+
+To move to a newer dexpp, update its version range in `dependencies` in
+`pyproject.toml` and run `uv lock`.
+
+To release: set `version` in `pyproject.toml`, turn the `## Unreleased`
+section of `CHANGELOG.md` into `## <version>`, and push a `v<version>` tag. The
+release workflow builds the wheel and the sdist, checks that the wheel installs
+and runs, creates the GitHub release with that changelog section as notes, and
+uploads the same files to PyPI.
+
+## Layout
+
+- `split_dex/cli.py` — the command-line interface.
+- `split_dex/splitter.py` — reads the APK, sorts the classes into sets, runs
+  each set's strategy, checks its groups, and writes the DEX files and maps.
+- `split_dex/class_sets.py` — sets, patterns, and how classes are assigned.
+- `split_dex/strategy.py` — the interface between the splitter and strategies.
+- `split_dex/registry.py` — finds strategies (built-in, plugin, module or file)
+  and creates them from their settings.
+- `split_dex/strategies/` — the built-in strategies.
+- `split_dex/partition.py` — the package-affinity grouping algorithm.
+- `tests/` — the unit tests (pytest).

split_dex-0.1.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+split_dex/__init__.py,sha256=yDE--EYxkoM-n19N4EXVitZtitbDlvpHBHu2dbVcdBs,139
+split_dex/class_sets.py,sha256=Ud6qt8p4epXDeUYyoaWTN6AsNYa8Jyaoa1ft5DV3Y8o,4450
+split_dex/cli.py,sha256=ZlvD4usAoGMMADdfV2wEeM6S6moP1Y2gyB1E9STJhog,8468
+split_dex/partition.py,sha256=wlLfSS08Pu92uXgJuwCQfFPoSBWUdxtpyVugYL4mPr4,8677
+split_dex/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+split_dex/registry.py,sha256=hDF5z50G4jjEJshC92OEeiAmk4mp8_MQwdIwDGQRJ5o,8195
+split_dex/splitter.py,sha256=z6AjipuSbwZLuAuHtwZ7_IzxabC84m6hCa4S17kcPew,22462
+split_dex/strategy.py,sha256=-QeGECv1G20QVQuDM_I08bncWyv4x4TcU3Gz-zAUC5I,7801
+split_dex/strategies/__init__.py,sha256=Uo3u3Zt2TKBn_cwXSXPeHx4nDJAs9UUARdgJHPO7Q6w,84
+split_dex/strategies/original.py,sha256=ND9DK9EbYDfuylnd0cs42RxS-n0hi85Bmi2757LOZCI,2252
+split_dex/strategies/package_affinity.py,sha256=dWycvxMUwflDnTwD4043V06gxJa2j52d39mCaCSzDLI,1296
+split_dex/strategies/package_order.py,sha256=KiI0cXbFfR1St846JTiDIqdITL5hRHDuVbuv1mrejZg,2022
+split_dex/strategies/single.py,sha256=6Ih8Jn28BnpO-VcLd8o5Fy1yuNJbavf0fg36gIyln4w,745
+split_dex-0.1.0.dist-info/METADATA,sha256=DFHwfYJCDUPeynDNoWbW04121mZVfq2ZSujMuRdwCXc,10556
+split_dex-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+split_dex-0.1.0.dist-info/entry_points.txt,sha256=qiaGTLQ0TUu5ts-cKQBGIe7XZDyylMP6XsCmFNl6Zbo,49
+split_dex-0.1.0.dist-info/licenses/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
+split_dex-0.1.0.dist-info/RECORD,,

split_dex-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

split_dex-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+split-dex = split_dex.cli:main