embedl-deploy 0.1.1__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {embedl_deploy-0.1.1/src/embedl_deploy.egg-info → embedl_deploy-0.2.0}/PKG-INFO +1 -1
- embedl_deploy-0.2.0/src/embedl_deploy/_internal/core/backend.py +125 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/core/match.py +74 -50
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/core/modules.py +20 -4
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/core/pattern.py +153 -48
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/core/plan.py +18 -14
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/core/quantize/__init__.py +1 -1
- embedl_deploy-0.2.0/src/embedl_deploy/_internal/core/quantize/calibrate.py +133 -0
- {embedl_deploy-0.1.1/src/embedl_deploy/_internal/core → embedl_deploy-0.2.0/src/embedl_deploy/_internal/core/quantize}/config.py +17 -57
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/core/quantize/main.py +72 -16
- embedl_deploy-0.2.0/src/embedl_deploy/_internal/core/quantize/prepare.py +52 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/core/quantize/qat.py +1 -1
- embedl_deploy-0.2.0/src/embedl_deploy/_internal/core/quantize/stubs.py +385 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/core/quantize/utils.py +12 -10
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/core/replace.py +3 -0
- embedl_deploy-0.2.0/src/embedl_deploy/_internal/tensorrt/backend.py +18 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/tensorrt/modules/attention.py +20 -14
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/tensorrt/modules/conv.py +6 -7
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/tensorrt/modules/linear.py +15 -14
- embedl_deploy-0.2.0/src/embedl_deploy/_internal/tensorrt/modules/pointwise.py +39 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/tensorrt/modules/swin_attention.py +3 -2
- embedl_deploy-0.2.0/src/embedl_deploy/_internal/tensorrt/patterns/conversions/__init__.py +15 -0
- embedl_deploy-0.1.1/src/embedl_deploy/_internal/tensorrt/patterns/conversions.py → embedl_deploy-0.2.0/src/embedl_deploy/_internal/tensorrt/patterns/conversions/attention.py +232 -254
- embedl_deploy-0.2.0/src/embedl_deploy/_internal/tensorrt/patterns/conversions/general.py +300 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/tensorrt/patterns/fusions/attention.py +0 -16
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/tensorrt/patterns/fusions/conv.py +10 -18
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/tensorrt/patterns/fusions/linear.py +7 -17
- embedl_deploy-0.2.0/src/embedl_deploy/_internal/tensorrt/patterns/fusions/pointwise.py +55 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/tensorrt/patterns/fusions/pool.py +0 -2
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/tensorrt/patterns/quantizations.py +14 -12
- embedl_deploy-0.2.0/src/embedl_deploy/_internal/tensorrt/patterns/recompositions.py +819 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/tensorrt/patterns/smoothings.py +17 -15
- embedl_deploy-0.2.0/src/embedl_deploy/_internal/tensorrt/patterns/utils.py +81 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/tensorrt/plan.py +40 -1
- embedl_deploy-0.2.0/src/embedl_deploy/backend/__init__.py +15 -0
- embedl_deploy-0.2.0/src/embedl_deploy/py.typed +0 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/quantize/__init__.py +10 -10
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/tensorrt/patterns/__init__.py +11 -1
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/version/public.py +1 -1
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0/src/embedl_deploy.egg-info}/PKG-INFO +1 -1
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy.egg-info/SOURCES.txt +15 -5
- embedl_deploy-0.1.1/src/embedl_deploy/_internal/core/quantize/modules.py +0 -254
- embedl_deploy-0.1.1/src/embedl_deploy/_internal/core/quantize/qdq.py +0 -101
- embedl_deploy-0.1.1/src/embedl_deploy/_internal/core/quantize/smooth_quant.py +0 -242
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/LICENSE +0 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/NOTICE +0 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/README.md +0 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/pyproject.toml +0 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/setup.cfg +0 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/__init__.py +0 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/__init__.py +0 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/core/__init__.py +0 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/tensorrt/__init__.py +0 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/tensorrt/modules/__init__.py +0 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/tensorrt/modules/pool.py +0 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/tensorrt/patterns/__init__.py +0 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/_internal/tensorrt/patterns/fusions/__init__.py +0 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/tensorrt/__init__.py +0 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/tensorrt/modules/__init__.py +0 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy/version/__init__.py +0 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy.egg-info/dependency_links.txt +0 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy.egg-info/requires.txt +0 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/src/embedl_deploy.egg-info/top_level.txt +0 -0
- {embedl_deploy-0.1.1 → embedl_deploy-0.2.0}/tests/test_version.py +0 -0
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
# Copyright (C) 2026 Embedl AB
|
|
2
|
+
|
|
3
|
+
"""Backend discovery and selection."""
|
|
4
|
+
|
|
5
|
+
import importlib
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from embedl_deploy._internal.core.pattern import Pattern
|
|
11
|
+
|
|
12
|
+
_INTERNAL_DIR = Path(__file__).resolve().parent.parent
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class Backend:
|
|
17
|
+
"""A collection of patterns for a specific hardware target."""
|
|
18
|
+
|
|
19
|
+
#: Structural rewrite patterns, applied iteratively.
|
|
20
|
+
conversion_patterns: Sequence[Pattern]
|
|
21
|
+
#: Fusion patterns, applied in a single pass after conversions.
|
|
22
|
+
fusion_patterns: Sequence[Pattern]
|
|
23
|
+
#: SmoothQuant preparation patterns.
|
|
24
|
+
smooth_patterns: Sequence[Pattern]
|
|
25
|
+
#: Q/DQ stub insertion patterns for quantisation.
|
|
26
|
+
quantized_patterns: Sequence[Pattern]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class _BackendState:
|
|
30
|
+
"""Module-level mutable state for backend discovery and selection."""
|
|
31
|
+
|
|
32
|
+
#: The currently selected backend.
|
|
33
|
+
backend: Backend | None = None
|
|
34
|
+
#: Cached discovery result.
|
|
35
|
+
backends: dict[str, Backend] | None = None
|
|
36
|
+
|
|
37
|
+
@classmethod
|
|
38
|
+
def reset(cls) -> None:
|
|
39
|
+
"""Clear cached discovery results and the active backend."""
|
|
40
|
+
cls.backend = None
|
|
41
|
+
cls.backends = None
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _discover_backends() -> dict[str, Backend]:
|
|
45
|
+
"""Scan ``_internal/`` for importable backend packages.
|
|
46
|
+
|
|
47
|
+
Each subdirectory (except ``core``) is tried as
|
|
48
|
+
``embedl_deploy._internal.<name>.backend``. Directories whose
|
|
49
|
+
module cannot be found are skipped; import errors from
|
|
50
|
+
transitive dependencies are propagated. Results are cached
|
|
51
|
+
after the first call.
|
|
52
|
+
|
|
53
|
+
:returns:
|
|
54
|
+
Mapping of backend name to ``Backend`` instance.
|
|
55
|
+
"""
|
|
56
|
+
backends = _BackendState.backends
|
|
57
|
+
if backends is None:
|
|
58
|
+
backends = {}
|
|
59
|
+
for entry in sorted(_INTERNAL_DIR.iterdir()):
|
|
60
|
+
if (
|
|
61
|
+
not entry.is_dir()
|
|
62
|
+
or entry.name.startswith("_")
|
|
63
|
+
or entry.name == "core"
|
|
64
|
+
):
|
|
65
|
+
continue
|
|
66
|
+
module_path = f"embedl_deploy._internal.{entry.name}.backend"
|
|
67
|
+
try:
|
|
68
|
+
mod = importlib.import_module(module_path)
|
|
69
|
+
except ModuleNotFoundError as e:
|
|
70
|
+
if e.name == module_path:
|
|
71
|
+
continue
|
|
72
|
+
raise
|
|
73
|
+
backend = getattr(mod, "BACKEND", None)
|
|
74
|
+
if isinstance(backend, Backend):
|
|
75
|
+
backends[entry.name] = backend
|
|
76
|
+
_BackendState.backends = backends
|
|
77
|
+
return backends
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def get_backend() -> Backend:
|
|
81
|
+
"""Return the active backend, discovering it if necessary.
|
|
82
|
+
|
|
83
|
+
If no backend has been set via :func:`set_backend`, the installed
|
|
84
|
+
backends are discovered automatically. When exactly one is found
|
|
85
|
+
it becomes the active backend.
|
|
86
|
+
|
|
87
|
+
:returns:
|
|
88
|
+
The active :class:`Backend`.
|
|
89
|
+
:raises RuntimeError:
|
|
90
|
+
If no backends are installed, or if multiple backends are
|
|
91
|
+
installed and none has been explicitly selected.
|
|
92
|
+
"""
|
|
93
|
+
backend = _BackendState.backend
|
|
94
|
+
if backend is None:
|
|
95
|
+
backends = _discover_backends()
|
|
96
|
+
if len(backends) == 0:
|
|
97
|
+
raise RuntimeError(
|
|
98
|
+
"No backends found — install at least one backend"
|
|
99
|
+
)
|
|
100
|
+
if len(backends) > 1:
|
|
101
|
+
names = ", ".join(sorted(backends))
|
|
102
|
+
raise RuntimeError(
|
|
103
|
+
f"Multiple backends found ({names}). "
|
|
104
|
+
"Call set_backend() to select one."
|
|
105
|
+
)
|
|
106
|
+
backend = next(iter(backends.values()))
|
|
107
|
+
_BackendState.backend = backend
|
|
108
|
+
return backend
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def set_backend(name: str) -> None:
|
|
112
|
+
"""Select the active backend by name.
|
|
113
|
+
|
|
114
|
+
:param name:
|
|
115
|
+
The name of a discovered backend (e.g. ``"tensorrt"``).
|
|
116
|
+
:raises ValueError:
|
|
117
|
+
If `name` does not match any installed backend.
|
|
118
|
+
"""
|
|
119
|
+
backends = _discover_backends()
|
|
120
|
+
if name not in backends:
|
|
121
|
+
available = ", ".join(sorted(backends)) or "(none)"
|
|
122
|
+
raise ValueError(
|
|
123
|
+
f"Backend {name!r} not found. " f"Available backends: {available}"
|
|
124
|
+
)
|
|
125
|
+
_BackendState.backend = backends[name]
|
|
@@ -9,9 +9,9 @@ occurrences of operation chains in ``torch.fx`` graphs.
|
|
|
9
9
|
"""
|
|
10
10
|
|
|
11
11
|
import itertools
|
|
12
|
-
|
|
12
|
+
import types
|
|
13
13
|
|
|
14
|
-
from torch import fx
|
|
14
|
+
from torch import fx
|
|
15
15
|
|
|
16
16
|
from embedl_deploy._internal.core.pattern import (
|
|
17
17
|
Fork,
|
|
@@ -19,6 +19,7 @@ from embedl_deploy._internal.core.pattern import (
|
|
|
19
19
|
NodeCheck,
|
|
20
20
|
Pattern,
|
|
21
21
|
PatternMatch,
|
|
22
|
+
SharedNodeCheck,
|
|
22
23
|
Tree,
|
|
23
24
|
TreeMatch,
|
|
24
25
|
Trunk,
|
|
@@ -26,28 +27,40 @@ from embedl_deploy._internal.core.pattern import (
|
|
|
26
27
|
get_module,
|
|
27
28
|
)
|
|
28
29
|
|
|
29
|
-
ActivationModules: TypeAlias = (
|
|
30
|
-
nn.ReLU
|
|
31
|
-
| nn.ReLU6
|
|
32
|
-
| nn.GELU
|
|
33
|
-
| nn.SiLU
|
|
34
|
-
| nn.Mish
|
|
35
|
-
| nn.Hardswish
|
|
36
|
-
| nn.Hardsigmoid
|
|
37
|
-
| nn.LeakyReLU
|
|
38
|
-
| nn.PReLU
|
|
39
|
-
| nn.ELU
|
|
40
|
-
| nn.Sigmoid
|
|
41
|
-
| nn.Tanh
|
|
42
|
-
)
|
|
43
30
|
|
|
44
|
-
|
|
45
|
-
|
|
31
|
+
class _SharedNodeCheckSession:
|
|
32
|
+
"""Checkpoint/rollback scope for ``SharedNodeCheck`` cache entries.
|
|
33
|
+
|
|
34
|
+
Between permutation attempts inside a fork, call :meth:`rollback`
|
|
35
|
+
explicitly. On normal exit (success) cache entries survive so that
|
|
36
|
+
enclosing sessions can still enforce cross-fork shared-node
|
|
37
|
+
constraints. On failure exit (no permutation matched and the block
|
|
38
|
+
falls through) ``__exit__`` rolls back automatically.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def __init__(self) -> None:
|
|
42
|
+
self._checkpoint = SharedNodeCheck.checkpoint()
|
|
43
|
+
self._succeeded = False
|
|
44
|
+
|
|
45
|
+
def __enter__(self) -> "_SharedNodeCheckSession":
|
|
46
|
+
return self
|
|
47
|
+
|
|
48
|
+
def success(self) -> None:
|
|
49
|
+
"""Mark this session as successful — skip rollback on exit."""
|
|
50
|
+
self._succeeded = True
|
|
51
|
+
|
|
52
|
+
def __exit__(self, *args: object) -> None:
|
|
53
|
+
if not self._succeeded:
|
|
54
|
+
self.rollback()
|
|
55
|
+
|
|
56
|
+
def rollback(self) -> None:
|
|
57
|
+
"""Reset every cache event logged since this session started."""
|
|
58
|
+
SharedNodeCheck.rollback_to(self._checkpoint)
|
|
46
59
|
|
|
47
60
|
|
|
48
61
|
def _node_matches(node: fx.Node, checks: ModType | NodeCheck) -> bool:
|
|
49
62
|
"""Return whether `node` satisfies `checks`."""
|
|
50
|
-
if isinstance(checks, (type,
|
|
63
|
+
if isinstance(checks, (type, types.UnionType)):
|
|
51
64
|
return isinstance(get_module(node), checks)
|
|
52
65
|
return checks(node)
|
|
53
66
|
|
|
@@ -133,34 +146,43 @@ def _match_fork_at(
|
|
|
133
146
|
return None
|
|
134
147
|
|
|
135
148
|
fork_node = trunk.pre_trunk_nodes[0]
|
|
136
|
-
if (
|
|
149
|
+
if getattr(fork.operator, "is_node_check", False):
|
|
150
|
+
if not fork.operator(fork_node):
|
|
151
|
+
return None
|
|
152
|
+
elif (
|
|
137
153
|
fork_node.op != "call_function"
|
|
138
154
|
or fork_node.target is not fork.operator
|
|
139
155
|
):
|
|
140
156
|
return None
|
|
141
157
|
|
|
142
158
|
args = [a for a in fork_node.args if isinstance(a, fx.Node)]
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
159
|
+
perms = (
|
|
160
|
+
itertools.permutations(range(len(fork.inputs)))
|
|
161
|
+
if fork.perms_override is None
|
|
162
|
+
else fork.perms_override
|
|
163
|
+
)
|
|
164
|
+
with _SharedNodeCheckSession() as session:
|
|
165
|
+
for perm in perms:
|
|
166
|
+
session.rollback()
|
|
167
|
+
if len(perm) != len(args):
|
|
168
|
+
continue
|
|
169
|
+
fork_matched = True
|
|
170
|
+
tree_matches = [TreeMatch() for _ in fork.inputs]
|
|
171
|
+
for arg_idx, input_idx in enumerate(perm):
|
|
172
|
+
tree_match = _match_tree_at(
|
|
173
|
+
args[arg_idx], fork.inputs[input_idx]
|
|
174
|
+
)
|
|
175
|
+
if tree_match is None:
|
|
176
|
+
fork_matched = False
|
|
177
|
+
break
|
|
178
|
+
tree_matches[input_idx] = tree_match
|
|
179
|
+
if fork_matched:
|
|
180
|
+
session.success()
|
|
181
|
+
return TreeMatch(
|
|
182
|
+
pre_trunk_nodes=[fork_node],
|
|
183
|
+
trunk_nodes=trunk.trunk_nodes,
|
|
184
|
+
nested=tree_matches,
|
|
185
|
+
)
|
|
164
186
|
return None
|
|
165
187
|
|
|
166
188
|
|
|
@@ -218,15 +240,17 @@ def match_tree(
|
|
|
218
240
|
raise ValueError("``pattern`` has no tree to match.")
|
|
219
241
|
|
|
220
242
|
matches: list[PatternMatch] = []
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
243
|
+
with _SharedNodeCheckSession() as session:
|
|
244
|
+
for node in reversed(list(graph_module.graph.nodes)):
|
|
245
|
+
session.rollback()
|
|
246
|
+
matched = _match_tree_at(node, tree)
|
|
247
|
+
if matched is not None:
|
|
248
|
+
matches.append(
|
|
249
|
+
PatternMatch(
|
|
250
|
+
pattern=pattern,
|
|
251
|
+
graph_module=graph_module,
|
|
252
|
+
tree_match=matched,
|
|
253
|
+
)
|
|
229
254
|
)
|
|
230
|
-
)
|
|
231
255
|
|
|
232
256
|
return matches
|
|
@@ -6,10 +6,26 @@
|
|
|
6
6
|
# torch lacks type stubs, so nn.Module resolves to Any.
|
|
7
7
|
|
|
8
8
|
from abc import ABC
|
|
9
|
+
from typing import TypeAlias
|
|
9
10
|
|
|
10
11
|
from torch import fx, nn
|
|
11
12
|
|
|
12
|
-
from embedl_deploy._internal.core.quantize.
|
|
13
|
+
from embedl_deploy._internal.core.quantize.stubs import QuantStub
|
|
14
|
+
|
|
15
|
+
ActivationLike: TypeAlias = (
|
|
16
|
+
nn.ReLU
|
|
17
|
+
| nn.ReLU6
|
|
18
|
+
| nn.GELU
|
|
19
|
+
| nn.SiLU
|
|
20
|
+
| nn.Mish
|
|
21
|
+
| nn.Hardswish
|
|
22
|
+
| nn.Hardsigmoid
|
|
23
|
+
| nn.LeakyReLU
|
|
24
|
+
| nn.PReLU
|
|
25
|
+
| nn.ELU
|
|
26
|
+
| nn.Sigmoid
|
|
27
|
+
| nn.Tanh
|
|
28
|
+
)
|
|
13
29
|
|
|
14
30
|
|
|
15
31
|
class ConvertedModule(nn.Module, ABC):
|
|
@@ -29,13 +45,13 @@ class FusedModule(nn.Module, ABC):
|
|
|
29
45
|
|
|
30
46
|
Backend packages (e.g. ``tensorrt``) subclass this for their concrete
|
|
31
47
|
fused modules (``FusedConvBNAct``, etc.). The generic Q/DQ insertion
|
|
32
|
-
pass in :mod:`~embedl_deploy._internal.core.quantize.
|
|
48
|
+
pass in :mod:`~embedl_deploy._internal.core.quantize.prepare` uses
|
|
33
49
|
``isinstance(mod, FusedModule)`` to identify fused nodes without
|
|
34
50
|
knowing backend-specific types.
|
|
35
51
|
"""
|
|
36
52
|
|
|
37
53
|
#: Positional argument indices that should receive a
|
|
38
|
-
#: :class:`~embedl_deploy._internal.core.quantize.
|
|
54
|
+
#: :class:`~embedl_deploy._internal.core.quantize.stubs.QuantStub`.
|
|
39
55
|
#: The Q/DQ insertion pass uses this to decide which inputs of the
|
|
40
56
|
#: fused node to quantize. Every subclass must set this explicitly.
|
|
41
57
|
inputs_to_quantize: set[int]
|
|
@@ -43,7 +59,7 @@ class FusedModule(nn.Module, ABC):
|
|
|
43
59
|
def __init__(self) -> None:
|
|
44
60
|
super().__init__()
|
|
45
61
|
#: Maps each index in :attr:`inputs_to_quantize` to a
|
|
46
|
-
#: :class:`~embedl_deploy._internal.core.quantize.
|
|
62
|
+
#: :class:`~embedl_deploy._internal.core.quantize.stubs.QuantStub`.
|
|
47
63
|
self.input_quant_stubs: dict[int, QuantStub] = {
|
|
48
64
|
idx: QuantStub({self}) for idx in self.inputs_to_quantize
|
|
49
65
|
}
|
|
@@ -9,9 +9,9 @@ methods — :meth:`~embedl_deploy._internal.core.pattern.Pattern.match` and
|
|
|
9
9
|
what to look for and how to rewrite the graph.
|
|
10
10
|
"""
|
|
11
11
|
|
|
12
|
-
import
|
|
12
|
+
import types
|
|
13
13
|
from abc import ABC, abstractmethod
|
|
14
|
-
from collections.abc import Callable
|
|
14
|
+
from collections.abc import Callable
|
|
15
15
|
from dataclasses import dataclass, field
|
|
16
16
|
from typing import Any, Literal, TypedDict, TypeVar, overload
|
|
17
17
|
|
|
@@ -41,7 +41,7 @@ def resolve_module(
|
|
|
41
41
|
:returns:
|
|
42
42
|
The ``nn.Module`` instance, narrowed to `cls`.
|
|
43
43
|
:raises LookupError:
|
|
44
|
-
If
|
|
44
|
+
If `node` is not a ``call_module`` node, the target module
|
|
45
45
|
cannot be found, or the module is not an instance of `cls`.
|
|
46
46
|
"""
|
|
47
47
|
if not (
|
|
@@ -72,8 +72,8 @@ def get_module(node: fx.Node) -> nn.Module | None:
|
|
|
72
72
|
return None
|
|
73
73
|
|
|
74
74
|
|
|
75
|
-
#: A single module type or a
|
|
76
|
-
ModType = type[nn.Module] |
|
|
75
|
+
#: A single module type or a union of module types for ``isinstance`` checks.
|
|
76
|
+
ModType = type[nn.Module] | types.UnionType
|
|
77
77
|
|
|
78
78
|
#: A callable predicate that inspects a node and returns whether it matches.
|
|
79
79
|
NodeCheck = Callable[[fx.Node], bool]
|
|
@@ -83,11 +83,12 @@ NodeCheck = Callable[[fx.Node], bool]
|
|
|
83
83
|
class Wildcard:
|
|
84
84
|
"""Zero or more consecutive nodes to absorb during matching.
|
|
85
85
|
|
|
86
|
-
When a :class
|
|
86
|
+
When a :class:`~embedl_deploy._internal.core.pattern.Wildcard` appears in a
|
|
87
87
|
:data:`~embedl_deploy._internal.core.pattern.Trunk`, the matcher walks
|
|
88
88
|
backward through nodes that satisfy `checks`. The absorbed nodes are
|
|
89
|
-
recorded in `nodes` and the
|
|
90
|
-
|
|
89
|
+
recorded in `nodes` and the
|
|
90
|
+
:class:`~embedl_deploy._internal.core.pattern.Wildcard` itself is stored as
|
|
91
|
+
an entry in
|
|
91
92
|
:attr:`~embedl_deploy._internal.core.pattern.TreeMatch.trunk_nodes`.
|
|
92
93
|
"""
|
|
93
94
|
|
|
@@ -101,7 +102,7 @@ class Wildcard:
|
|
|
101
102
|
|
|
102
103
|
@property
|
|
103
104
|
def name(self) -> list[str]:
|
|
104
|
-
"""Node names of absorbed nodes, for use in
|
|
105
|
+
"""Node names of absorbed nodes, for use in serialization."""
|
|
105
106
|
return [n.name for n in self.nodes]
|
|
106
107
|
|
|
107
108
|
|
|
@@ -110,7 +111,68 @@ class Wildcard:
|
|
|
110
111
|
#: (for ``isinstance`` checks), a
|
|
111
112
|
#: :data:`~embedl_deploy._internal.core.pattern.NodeCheck` predicate,
|
|
112
113
|
#: or a :class:`~embedl_deploy._internal.core.pattern.Wildcard`.
|
|
113
|
-
Trunk =
|
|
114
|
+
Trunk = tuple[ModType | NodeCheck | Wildcard, ...]
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def node_check(
|
|
118
|
+
fn: Callable[[fx.Node], bool],
|
|
119
|
+
) -> Callable[[fx.Node], bool]:
|
|
120
|
+
"""Mark a function as a ``Fork.operator`` predicate.
|
|
121
|
+
|
|
122
|
+
When the matcher encounters a
|
|
123
|
+
:class:`~embedl_deploy._internal.core.pattern.Fork` whose ``operator``
|
|
124
|
+
carries the ``is_node_check`` attribute, it calls the operator with the
|
|
125
|
+
candidate node and expects a ``bool``. Without the marker the operator is
|
|
126
|
+
identity-compared against ``node.target``.
|
|
127
|
+
|
|
128
|
+
:param fn:
|
|
129
|
+
A :data:`~embedl_deploy._internal.core.pattern.NodeCheck` callable.
|
|
130
|
+
:returns:
|
|
131
|
+
The same callable, with ``is_node_check = True`` attached.
|
|
132
|
+
"""
|
|
133
|
+
setattr(fn, "is_node_check", True)
|
|
134
|
+
return fn
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class SharedNodeCheck:
|
|
138
|
+
"""Constrain multiple fork branches to the same physical node.
|
|
139
|
+
|
|
140
|
+
On the first call, delegates to an inner ``NodeCheck`` and caches the
|
|
141
|
+
matched node. Subsequent calls succeed only when the candidate is the
|
|
142
|
+
exact same ``fx.Node`` instance. A class-level log records every
|
|
143
|
+
cache event so the matcher can checkpoint and rollback.
|
|
144
|
+
"""
|
|
145
|
+
|
|
146
|
+
_log: list["SharedNodeCheck"] = []
|
|
147
|
+
|
|
148
|
+
def __init__(self, inner: NodeCheck) -> None:
|
|
149
|
+
self._inner = inner
|
|
150
|
+
self._cached: fx.Node | None = None
|
|
151
|
+
|
|
152
|
+
def __call__(self, node: fx.Node) -> bool:
|
|
153
|
+
if self._cached is None:
|
|
154
|
+
if not self._inner(node):
|
|
155
|
+
return False
|
|
156
|
+
self._cached = node
|
|
157
|
+
SharedNodeCheck._log.append(self)
|
|
158
|
+
return True
|
|
159
|
+
return node is self._cached
|
|
160
|
+
|
|
161
|
+
def reset(self) -> None:
|
|
162
|
+
"""Clear the cached node."""
|
|
163
|
+
self._cached = None
|
|
164
|
+
|
|
165
|
+
@classmethod
|
|
166
|
+
def checkpoint(cls) -> int:
|
|
167
|
+
"""Return a rollback point for use with ``rollback_to``."""
|
|
168
|
+
return len(cls._log)
|
|
169
|
+
|
|
170
|
+
@classmethod
|
|
171
|
+
def rollback_to(cls, checkpoint: int) -> None:
|
|
172
|
+
"""Reset and discard every log entry from `checkpoint` onward."""
|
|
173
|
+
for entry in cls._log[checkpoint:]:
|
|
174
|
+
entry.reset()
|
|
175
|
+
del cls._log[checkpoint:]
|
|
114
176
|
|
|
115
177
|
|
|
116
178
|
@dataclass(frozen=True)
|
|
@@ -125,12 +187,71 @@ class Fork:
|
|
|
125
187
|
#: is itself a :data:`~embedl_deploy._internal.core.pattern.Tree`
|
|
126
188
|
#: (a :data:`~embedl_deploy._internal.core.pattern.Trunk` or
|
|
127
189
|
#: another :class:`~embedl_deploy._internal.core.pattern.Fork`).
|
|
128
|
-
#: An empty
|
|
129
|
-
inputs:
|
|
130
|
-
#: The function that the
|
|
190
|
+
#: An empty tuple means the branch accepts any single node.
|
|
191
|
+
inputs: tuple["Tree", ...]
|
|
192
|
+
#: The function that the fork node must invoke. There are two
|
|
193
|
+
#: modes, selected by the presence of the ``is_node_check``
|
|
194
|
+
#: attribute (set by the :func:`node_check` decorator):
|
|
195
|
+
#:
|
|
196
|
+
#: **Identity compare** (default) — the fork node's ``target``
|
|
197
|
+
#: is checked with ``is`` against this value. Use for concrete
|
|
198
|
+
#: functions like ``operator.add`` or
|
|
199
|
+
#: ``torch.ops.aten.conv2d.default``.
|
|
200
|
+
#:
|
|
201
|
+
#: **Predicate** (``@node_check``) — called with the fork node;
|
|
202
|
+
#: must return ``True`` for the match to succeed. Use when a
|
|
203
|
+
#: single function target is not sufficient, e.g. when the match
|
|
204
|
+
#: also depends on argument shapes or metadata.
|
|
131
205
|
operator: Callable[..., Any]
|
|
132
206
|
#: The straight chain of modules after the merge node.
|
|
133
207
|
output: Trunk
|
|
208
|
+
#: Optional override of the permitted mappings from the arguments of the
|
|
209
|
+
#: ``operator`` to the branches in ``inputs``. Only ``fx.Node`` arguments
|
|
210
|
+
#: are considered (``None`` and scalar arguments are ignored). Each
|
|
211
|
+
#: mapping is a tuple whose length equals the expected number of such
|
|
212
|
+
#: arguments. At index *i*, a value of *k* in a mapping means that the
|
|
213
|
+
#: *i*-th argument of the ``operator`` must match the *k*-th branch in
|
|
214
|
+
#: ``inputs``. A mapping may not be longer than ``inputs``, but it may be
|
|
215
|
+
#: shorter if some of the branches are optional.
|
|
216
|
+
#:
|
|
217
|
+
#: .. code-block:: python
|
|
218
|
+
#:
|
|
219
|
+
#: fork = Fork(
|
|
220
|
+
#: inputs=(
|
|
221
|
+
#: (), # input tensor
|
|
222
|
+
#: (_is_attr("weight"),),
|
|
223
|
+
#: (_is_attr("bias"),), # optional
|
|
224
|
+
#: ),
|
|
225
|
+
#: operator=torch.ops.aten.conv2d.default,
|
|
226
|
+
#: output=(),
|
|
227
|
+
#: perms_override=((0, 1, 2), (0, 1)),
|
|
228
|
+
#: )
|
|
229
|
+
#:
|
|
230
|
+
#: The two entries in ``perms_override`` cover the two shapes of the
|
|
231
|
+
#: ``aten.conv2d.default`` call: ``(0, 1, 2)`` matches when bias is
|
|
232
|
+
#: present, ``(0, 1)`` matches when it is absent.
|
|
233
|
+
#:
|
|
234
|
+
#: By default the ``perms_override`` is ``None``. In that case, the
|
|
235
|
+
#: arguments of the ``operator`` are checked against all full-length
|
|
236
|
+
#: permutations of the branches in ``inputs`` until one matches.
|
|
237
|
+
perms_override: tuple[tuple[int, ...], ...] | None = None
|
|
238
|
+
|
|
239
|
+
def __post_init__(self) -> None:
|
|
240
|
+
if self.perms_override is None:
|
|
241
|
+
return
|
|
242
|
+
valid_set = set(range(len(self.inputs)))
|
|
243
|
+
for perm in self.perms_override:
|
|
244
|
+
perm_set = set(perm)
|
|
245
|
+
if not perm_set <= valid_set:
|
|
246
|
+
raise ValueError(
|
|
247
|
+
f"perms_override entry {perm!r} contains "
|
|
248
|
+
f"out-of-range indices"
|
|
249
|
+
)
|
|
250
|
+
if len(perm_set) < len(perm):
|
|
251
|
+
raise ValueError(
|
|
252
|
+
f"perms_override entry {perm!r} contains "
|
|
253
|
+
f"duplicate indices"
|
|
254
|
+
)
|
|
134
255
|
|
|
135
256
|
|
|
136
257
|
#: A pattern topology: either a straight
|
|
@@ -139,24 +260,6 @@ class Fork:
|
|
|
139
260
|
Tree = Trunk | Fork
|
|
140
261
|
|
|
141
262
|
|
|
142
|
-
class QDQPoint(enum.Enum):
|
|
143
|
-
"""Named positions where quantization stubs are inserted.
|
|
144
|
-
|
|
145
|
-
Each :class:`~embedl_deploy._internal.core.pattern.Pattern` subclass
|
|
146
|
-
declares which Q/DQ points it needs via its
|
|
147
|
-
:attr:`~embedl_deploy._internal.core.pattern.Pattern.qdq_points` class
|
|
148
|
-
attribute. The Q/DQ insertion pass uses these declarations to place
|
|
149
|
-
:class:`~embedl_deploy._internal.core.quantize.modules.QuantStub` nodes at
|
|
150
|
-
the correct positions in the graph.
|
|
151
|
-
"""
|
|
152
|
-
|
|
153
|
-
INPUT = "input"
|
|
154
|
-
OUTPUT = "output"
|
|
155
|
-
RESIDUAL_INPUT = "residual_input"
|
|
156
|
-
KEY_INPUT = "key_input"
|
|
157
|
-
VALUE_INPUT = "value_input"
|
|
158
|
-
|
|
159
|
-
|
|
160
263
|
class Pattern(ABC):
|
|
161
264
|
"""A graph transformation rule: find a sub-graph and replace it.
|
|
162
265
|
|
|
@@ -179,12 +282,10 @@ class Pattern(ABC):
|
|
|
179
282
|
"""If ``True``, this pattern is a structural conversion that must
|
|
180
283
|
be applied before fusion matching."""
|
|
181
284
|
|
|
182
|
-
|
|
183
|
-
"""
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
:class:`~embedl_deploy._internal.core.quantize.modules.QuantStub`
|
|
187
|
-
nodes should be placed. The default (empty) means no Q/DQ stubs."""
|
|
285
|
+
symbolic_trace_only: bool = False
|
|
286
|
+
"""If ``True``, this pattern removes nodes that are artifacts of
|
|
287
|
+
``symbolic_trace``. This pattern has no effect on graphs exported with
|
|
288
|
+
``torch.export`` because the nodes never appear in those graphs."""
|
|
188
289
|
|
|
189
290
|
@abstractmethod
|
|
190
291
|
def match(self, graph_module: fx.GraphModule) -> list["PatternMatch"]:
|
|
@@ -215,10 +316,10 @@ class Pattern(ABC):
|
|
|
215
316
|
"""
|
|
216
317
|
|
|
217
318
|
|
|
218
|
-
class
|
|
219
|
-
"""JSON-
|
|
319
|
+
class SerializedTreeMatch(TypedDict):
|
|
320
|
+
"""JSON-serializable representation of a ``TreeMatch``."""
|
|
220
321
|
|
|
221
|
-
nested: list["
|
|
322
|
+
nested: list["SerializedTreeMatch"]
|
|
222
323
|
pre_trunk_nodes: list[str]
|
|
223
324
|
trunk_nodes: list[str | list[str]]
|
|
224
325
|
|
|
@@ -247,6 +348,7 @@ class TreeMatch:
|
|
|
247
348
|
trunk_nodes: list[fx.Node | Wildcard] = field(default_factory=list)
|
|
248
349
|
#: Per-branch sub-matches, indexed the same as
|
|
249
350
|
#: :attr:`~embedl_deploy._internal.core.pattern.Fork.inputs`.
|
|
351
|
+
#: Empty entries represent optional branches that were absent.
|
|
250
352
|
nested: list["TreeMatch"] = field(default_factory=list)
|
|
251
353
|
|
|
252
354
|
def get_input_nodes(self) -> tuple[fx.Node, ...]:
|
|
@@ -311,9 +413,10 @@ class TreeMatch:
|
|
|
311
413
|
|
|
312
414
|
With a single `index`, returns ``trunk_nodes[index]``. For a
|
|
313
415
|
:class:`~embedl_deploy._internal.core.pattern.Wildcard` position the
|
|
314
|
-
returned value is the
|
|
315
|
-
|
|
316
|
-
|
|
416
|
+
returned value is the
|
|
417
|
+
:class:`~embedl_deploy._internal.core.pattern.Wildcard` itself
|
|
418
|
+
(access its ``nodes`` attribute for the absorbed nodes). With
|
|
419
|
+
additional indices the lookup descends into
|
|
317
420
|
:attr:`~embedl_deploy._internal.core.pattern.TreeMatch.nested`
|
|
318
421
|
sub-matches first.
|
|
319
422
|
|
|
@@ -325,10 +428,12 @@ class TreeMatch:
|
|
|
325
428
|
:param args:
|
|
326
429
|
Additional indices forwarded recursively into nested sub-matches.
|
|
327
430
|
:param is_wildcard:
|
|
328
|
-
If ``True``, assert the entry is a
|
|
431
|
+
If ``True``, assert the entry is a
|
|
432
|
+
:class:`~embedl_deploy._internal.core.pattern.Wildcard` and return
|
|
329
433
|
it. If ``False`` (default), assert it is an ``fx.Node``.
|
|
330
434
|
:returns:
|
|
331
|
-
The matched ``fx.Node`` or
|
|
435
|
+
The matched ``fx.Node`` or
|
|
436
|
+
:class:`~embedl_deploy._internal.core.pattern.Wildcard`.
|
|
332
437
|
"""
|
|
333
438
|
if args:
|
|
334
439
|
nested = self.nested[index]
|
|
@@ -339,10 +444,10 @@ class TreeMatch:
|
|
|
339
444
|
assert isinstance(entry, Wildcard if is_wildcard else fx.Node)
|
|
340
445
|
return entry
|
|
341
446
|
|
|
342
|
-
def
|
|
343
|
-
"""
|
|
447
|
+
def serialize(self) -> SerializedTreeMatch:
|
|
448
|
+
"""Serialize this match to a JSON-serializable dict."""
|
|
344
449
|
return {
|
|
345
|
-
"nested": [nm.
|
|
450
|
+
"nested": [nm.serialize() for nm in self.nested],
|
|
346
451
|
"pre_trunk_nodes": [n.name for n in self.pre_trunk_nodes],
|
|
347
452
|
"trunk_nodes": [n.name for n in self.trunk_nodes],
|
|
348
453
|
}
|