codegen-2 2.16.1__tar.gz → 2.16.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {codegen_2-2.16.1 → codegen_2-2.16.2}/PKG-INFO +1 -4
- {codegen_2-2.16.1 → codegen_2-2.16.2}/codegen/models/__init__.py +2 -2
- {codegen_2-2.16.1 → codegen_2-2.16.2}/codegen/models/ast.py +27 -7
- {codegen_2-2.16.1 → codegen_2-2.16.2}/codegen/models/program.py +127 -12
- {codegen_2-2.16.1 → codegen_2-2.16.2}/codegen/models/types.py +2 -2
- codegen_2-2.16.2/codegen/models/utils.py +9 -0
- {codegen_2-2.16.1 → codegen_2-2.16.2}/codegen/models/var.py +12 -3
- {codegen_2-2.16.1 → codegen_2-2.16.2}/pyproject.toml +7 -4
- {codegen_2-2.16.1 → codegen_2-2.16.2}/README.md +0 -0
- {codegen_2-2.16.1 → codegen_2-2.16.2}/codegen/__init__.py +0 -0
- {codegen_2-2.16.1 → codegen_2-2.16.2}/codegen/models/expr.py +0 -0
- {codegen_2-2.16.1 → codegen_2-2.16.2}/codegen/models/statement.py +0 -0
|
@@ -1,14 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: codegen-2
|
|
3
|
-
Version: 2.16.
|
|
3
|
+
Version: 2.16.2
|
|
4
4
|
Summary: Meta-programming
|
|
5
5
|
Author: Binh Vu
|
|
6
6
|
Author-email: Binh Vu <binh@toan2.com>
|
|
7
7
|
License: MIT
|
|
8
|
-
Requires-Dist: pytest>=8.0.0 ; extra == 'dev'
|
|
9
|
-
Requires-Dist: autoflake>=2.2.1 ; extra == 'dev'
|
|
10
8
|
Requires-Python: >=3.11
|
|
11
|
-
Provides-Extra: dev
|
|
12
9
|
Description-Content-Type: text/markdown
|
|
13
10
|
|
|
14
11
|
# codegen
|
|
@@ -3,7 +3,7 @@ import codegen.models.statement as stmt
|
|
|
3
3
|
from codegen.models.ast import AST
|
|
4
4
|
from codegen.models.expr import PredefinedFn
|
|
5
5
|
from codegen.models.program import ImportHelper, Program
|
|
6
|
-
from codegen.models.types import
|
|
6
|
+
from codegen.models.types import KEY, AstID
|
|
7
7
|
from codegen.models.var import DeferredVar, Var, VarScope
|
|
8
8
|
|
|
9
9
|
__all__ = [
|
|
@@ -15,7 +15,7 @@ __all__ = [
|
|
|
15
15
|
"Program",
|
|
16
16
|
"stmt",
|
|
17
17
|
"PredefinedFn",
|
|
18
|
-
"
|
|
18
|
+
"AstID",
|
|
19
19
|
"KEY",
|
|
20
20
|
"ImportHelper",
|
|
21
21
|
]
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import copy
|
|
3
4
|
from dataclasses import dataclass, field
|
|
4
5
|
from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, Sequence
|
|
5
6
|
|
|
@@ -24,7 +25,8 @@ from codegen.models.statement import (
|
|
|
24
25
|
Statement,
|
|
25
26
|
TryStatement,
|
|
26
27
|
)
|
|
27
|
-
from codegen.models.types import
|
|
28
|
+
from codegen.models.types import AstID
|
|
29
|
+
from codegen.models.utils import move_ast_id
|
|
28
30
|
from codegen.models.var import DeferredVar, Var, VarScope
|
|
29
31
|
|
|
30
32
|
if TYPE_CHECKING:
|
|
@@ -33,7 +35,7 @@ if TYPE_CHECKING:
|
|
|
33
35
|
|
|
34
36
|
@dataclass
|
|
35
37
|
class AST:
|
|
36
|
-
id:
|
|
38
|
+
id: AstID
|
|
37
39
|
prog: Program
|
|
38
40
|
stmt: Statement
|
|
39
41
|
children: list[AST] = field(default_factory=list)
|
|
@@ -77,6 +79,24 @@ class AST:
|
|
|
77
79
|
for child in self.children:
|
|
78
80
|
child.freeze()
|
|
79
81
|
|
|
82
|
+
def clone(self, program: Program, var_memo: Optional[dict[int, Any]] = None) -> AST:
|
|
83
|
+
"""Create a deep copy of this AST and all its children.
|
|
84
|
+
|
|
85
|
+
The cloned AST shares the same program reference and preserves
|
|
86
|
+
all IDs, but statements and children are independent copies that
|
|
87
|
+
can be modified without affecting the original.
|
|
88
|
+
"""
|
|
89
|
+
ast = AST(self.id, program, copy.deepcopy(self.stmt, memo=var_memo))
|
|
90
|
+
ast._is_frozen = self._is_frozen
|
|
91
|
+
ast.children = [child.clone(program, var_memo) for child in self.children]
|
|
92
|
+
return ast
|
|
93
|
+
|
|
94
|
+
def move(self, source: AstID, target: AstID) -> AST:
|
|
95
|
+
self.id = move_ast_id(self.id, source, target)
|
|
96
|
+
for child in self.children:
|
|
97
|
+
child.move(source, target)
|
|
98
|
+
return self
|
|
99
|
+
|
|
80
100
|
def import_(self, module: str, is_import_attr: bool, alias: Optional[str] = None):
|
|
81
101
|
self._add_stmt(ImportStatement(module, is_import_attr, alias))
|
|
82
102
|
|
|
@@ -220,7 +240,7 @@ class AST:
|
|
|
220
240
|
ast, context, stop = fn(ast, context)
|
|
221
241
|
return ast
|
|
222
242
|
|
|
223
|
-
def has_statement_between_ast(self, stmtcls: type[Statement], end_ast_id:
|
|
243
|
+
def has_statement_between_ast(self, stmtcls: type[Statement], end_ast_id: AstID):
|
|
224
244
|
"""Check if there is a statement of the given type the current AST and the end_ast (exclusive)"""
|
|
225
245
|
for ast in self.children:
|
|
226
246
|
for intermediate_ast in ast.find_ast_to(end_ast_id):
|
|
@@ -231,7 +251,7 @@ class AST:
|
|
|
231
251
|
return True
|
|
232
252
|
return False
|
|
233
253
|
|
|
234
|
-
def find_ast_to(self, id:
|
|
254
|
+
def find_ast_to(self, id: AstID):
|
|
235
255
|
"""Iterate through ASTs that lead to the AST with the given id (inclusive)"""
|
|
236
256
|
if self.id == id:
|
|
237
257
|
yield self
|
|
@@ -242,7 +262,7 @@ class AST:
|
|
|
242
262
|
yield from ast.find_ast_to(id)
|
|
243
263
|
break
|
|
244
264
|
|
|
245
|
-
def find_ast(self, id:
|
|
265
|
+
def find_ast(self, id: AstID) -> Optional[AST]:
|
|
246
266
|
"""Find the AST with the given id"""
|
|
247
267
|
if len(self.id) > len(id):
|
|
248
268
|
# the ast that we are looking for is not in the subtree of this ast
|
|
@@ -261,11 +281,11 @@ class AST:
|
|
|
261
281
|
ast = ast.children[id[i]]
|
|
262
282
|
return ast
|
|
263
283
|
|
|
264
|
-
def next_child_id(self) ->
|
|
284
|
+
def next_child_id(self) -> AstID:
|
|
265
285
|
"""Get ID for the next child of this AST"""
|
|
266
286
|
return self.id + (len(self.children),)
|
|
267
287
|
|
|
268
|
-
def next_grandchild_id(self) ->
|
|
288
|
+
def next_grandchild_id(self) -> AstID:
|
|
269
289
|
"""Get ID for the next grandchild of this AST"""
|
|
270
290
|
return self.id + (len(self.children), 0)
|
|
271
291
|
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
-
import
|
|
4
|
-
from dataclasses import dataclass, field
|
|
5
|
-
from typing import Optional
|
|
3
|
+
import copy
|
|
4
|
+
from dataclasses import dataclass, field, fields, is_dataclass
|
|
5
|
+
from typing import Any, Optional
|
|
6
6
|
|
|
7
7
|
from codegen.models.ast import AST
|
|
8
8
|
from codegen.models.expr import ExprIdent # You may need to adjust this import path
|
|
9
9
|
from codegen.models.statement import BlockStatement, IfStatement
|
|
10
|
-
from codegen.models.types import
|
|
10
|
+
from codegen.models.types import KEY, AstID
|
|
11
11
|
from codegen.models.var import Var, VarScope
|
|
12
12
|
|
|
13
13
|
|
|
@@ -29,13 +29,13 @@ class Program:
|
|
|
29
29
|
self.import_area.import_(module, is_import_attr, alias)
|
|
30
30
|
self.imported_modules.add(module)
|
|
31
31
|
|
|
32
|
-
def get_ast_by_id(self, id:
|
|
32
|
+
def get_ast_by_id(self, id: AstID) -> AST:
|
|
33
33
|
ast = self.root
|
|
34
34
|
for item in id:
|
|
35
35
|
ast = ast.children[item]
|
|
36
36
|
return ast
|
|
37
37
|
|
|
38
|
-
def is_within_scope(self, ast:
|
|
38
|
+
def is_within_scope(self, ast: AstID, scope: VarScope) -> bool:
|
|
39
39
|
if len(ast) - 1 < len(scope.ast):
|
|
40
40
|
return False
|
|
41
41
|
|
|
@@ -56,7 +56,7 @@ class Program:
|
|
|
56
56
|
self,
|
|
57
57
|
name: str,
|
|
58
58
|
key: KEY,
|
|
59
|
-
ast:
|
|
59
|
+
ast: AstID,
|
|
60
60
|
force_name: Optional[str] = None,
|
|
61
61
|
type: Optional[ExprIdent] = None,
|
|
62
62
|
) -> Var:
|
|
@@ -74,7 +74,7 @@ class Program:
|
|
|
74
74
|
self,
|
|
75
75
|
*,
|
|
76
76
|
key: KEY,
|
|
77
|
-
at:
|
|
77
|
+
at: AstID,
|
|
78
78
|
) -> Var:
|
|
79
79
|
"""De-reference a variable from the memory that is available at the provided AST"""
|
|
80
80
|
reg = self.vars.find(key, at)
|
|
@@ -84,6 +84,82 @@ class Program:
|
|
|
84
84
|
name=reg.name, key=key, register_id=reg.id, scope=reg.scope, type=reg.type
|
|
85
85
|
)
|
|
86
86
|
|
|
87
|
+
def concatenate(self, *others: Program) -> Program:
|
|
88
|
+
"""Return a new program containing this program followed by the others."""
|
|
89
|
+
program = Program()
|
|
90
|
+
program.extend(self, *others)
|
|
91
|
+
return program
|
|
92
|
+
|
|
93
|
+
def extend(self, *others: Program) -> None:
|
|
94
|
+
"""Append other programs to this program."""
|
|
95
|
+
for source in others:
|
|
96
|
+
if source is self:
|
|
97
|
+
# Snapshot self-extension so we don't iterate source lists while appending to them.
|
|
98
|
+
source = source.concatenate()
|
|
99
|
+
|
|
100
|
+
body_child_offset = len(self.root.children) - 1
|
|
101
|
+
register_id_map = self.vars.extend(source.vars, body_child_offset)
|
|
102
|
+
|
|
103
|
+
var_memo: dict[int, Var] = {}
|
|
104
|
+
for var in self._collect_program_vars(source):
|
|
105
|
+
new_reg = register_id_map[var.register_id]
|
|
106
|
+
var_memo[id(var)] = Var(
|
|
107
|
+
name=var.name,
|
|
108
|
+
key=var.key,
|
|
109
|
+
register_id=new_reg.id,
|
|
110
|
+
scope=new_reg.scope,
|
|
111
|
+
force_name=var.force_name,
|
|
112
|
+
type=copy.deepcopy(var.type),
|
|
113
|
+
)
|
|
114
|
+
|
|
115
|
+
self._copy_children(source.import_area, self.import_area, var_memo)
|
|
116
|
+
for child in source.root.children:
|
|
117
|
+
if child is source.import_area:
|
|
118
|
+
continue
|
|
119
|
+
self.root.children.append(
|
|
120
|
+
child.clone(self, var_memo).move(child.id, self.root.next_child_id())
|
|
121
|
+
)
|
|
122
|
+
self.imported_modules.update(source.imported_modules)
|
|
123
|
+
|
|
124
|
+
def _copy_children(self, source_parent: AST, dest_parent: AST, var_memo: Optional[dict[int, Any]] = None):
|
|
125
|
+
for child in source_parent.children:
|
|
126
|
+
dest_parent.children.append(
|
|
127
|
+
child.clone(self, var_memo).move(source_parent.id, dest_parent.next_child_id())
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
@classmethod
|
|
131
|
+
def _collect_program_vars(cls, program: Program) -> list[Var]:
|
|
132
|
+
vars: list[Var] = []
|
|
133
|
+
cls._collect_ast_vars(program.root, vars)
|
|
134
|
+
return vars
|
|
135
|
+
|
|
136
|
+
@classmethod
|
|
137
|
+
def _collect_ast_vars(cls, ast: AST, vars: list[Var]):
|
|
138
|
+
cls._collect_vars(ast.stmt, vars)
|
|
139
|
+
for child in ast.children:
|
|
140
|
+
cls._collect_ast_vars(child, vars)
|
|
141
|
+
|
|
142
|
+
@classmethod
|
|
143
|
+
def _collect_vars(cls, obj: Any, vars: list[Var]):
|
|
144
|
+
if isinstance(obj, Var):
|
|
145
|
+
vars.append(obj)
|
|
146
|
+
return
|
|
147
|
+
|
|
148
|
+
if is_dataclass(obj) and not isinstance(obj, type):
|
|
149
|
+
for field in fields(obj):
|
|
150
|
+
cls._collect_vars(getattr(obj, field.name), vars)
|
|
151
|
+
return
|
|
152
|
+
|
|
153
|
+
if isinstance(obj, dict):
|
|
154
|
+
for key, value in obj.items():
|
|
155
|
+
cls._collect_vars(key, vars)
|
|
156
|
+
cls._collect_vars(value, vars)
|
|
157
|
+
return
|
|
158
|
+
|
|
159
|
+
if isinstance(obj, (list, tuple, set, frozenset)):
|
|
160
|
+
for item in obj:
|
|
161
|
+
cls._collect_vars(item, vars)
|
|
162
|
+
|
|
87
163
|
|
|
88
164
|
@dataclass
|
|
89
165
|
class VarRegister:
|
|
@@ -101,11 +177,43 @@ class VarRegisters:
|
|
|
101
177
|
registers: list[VarRegister] = field(default_factory=list)
|
|
102
178
|
key2registers: dict[KEY, list[int]] = field(default_factory=dict)
|
|
103
179
|
|
|
180
|
+
def extend(
|
|
181
|
+
self, source: VarRegisters, body_child_offset: int
|
|
182
|
+
) -> dict[int, VarRegister]:
|
|
183
|
+
register_id_map: dict[int, VarRegister] = {}
|
|
184
|
+
|
|
185
|
+
for reg in source.registers:
|
|
186
|
+
if len(reg.scope.ast) == 0:
|
|
187
|
+
new_scope = VarScope(
|
|
188
|
+
ast=(),
|
|
189
|
+
child_index_start=reg.scope.child_index_start + body_child_offset,
|
|
190
|
+
child_index_end=(reg.scope.child_index_end + body_child_offset) if reg.scope.child_index_end is not None else None
|
|
191
|
+
)
|
|
192
|
+
else:
|
|
193
|
+
new_scope = reg.scope.move(
|
|
194
|
+
(reg.scope.ast[0],),
|
|
195
|
+
(reg.scope.ast[0] + body_child_offset,)
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
new_reg = VarRegister(
|
|
199
|
+
id=len(self.registers),
|
|
200
|
+
name=reg.name,
|
|
201
|
+
key=reg.key,
|
|
202
|
+
scope=new_scope,
|
|
203
|
+
force_name=reg.force_name,
|
|
204
|
+
type=copy.deepcopy(reg.type),
|
|
205
|
+
)
|
|
206
|
+
self.registers.append(new_reg)
|
|
207
|
+
self.key2registers.setdefault(new_reg.key, []).append(new_reg.id)
|
|
208
|
+
register_id_map[reg.id] = new_reg
|
|
209
|
+
|
|
210
|
+
return register_id_map
|
|
211
|
+
|
|
104
212
|
def register(
|
|
105
213
|
self,
|
|
106
214
|
name: str,
|
|
107
215
|
key: KEY,
|
|
108
|
-
ast:
|
|
216
|
+
ast: AstID,
|
|
109
217
|
force_name: Optional[str] = None,
|
|
110
218
|
type: Optional[ExprIdent] = None,
|
|
111
219
|
) -> VarRegister:
|
|
@@ -135,22 +243,29 @@ class VarRegisters:
|
|
|
135
243
|
self.key2registers.setdefault(key, []).append(reg.id)
|
|
136
244
|
return reg
|
|
137
245
|
|
|
138
|
-
def find(self, key: KEY, ast:
|
|
246
|
+
def find(self, key: KEY, ast: AstID) -> Optional[VarRegister]:
|
|
139
247
|
"""Find the most specific register by name, key that is available in the given ast. If
|
|
140
|
-
there are multiple matches, the most specific register is the one with the largest depth
|
|
248
|
+
there are multiple matches, the most specific register is the one with the largest depth,
|
|
249
|
+
then the latest start point.
|
|
141
250
|
"""
|
|
142
251
|
if key not in self.key2registers:
|
|
143
252
|
return None
|
|
144
253
|
|
|
145
254
|
matched_regid = None
|
|
146
255
|
matched_regid_depth = -1
|
|
256
|
+
matched_regid_start: AstID = tuple()
|
|
147
257
|
for regid in self.key2registers[key]:
|
|
148
258
|
reg = self.registers[regid]
|
|
149
259
|
if self.program.is_within_scope(ast, reg.scope):
|
|
150
260
|
regscope_depth = reg.scope.get_depth()
|
|
151
|
-
|
|
261
|
+
regscope_start = reg.scope.ast + (reg.scope.child_index_start,)
|
|
262
|
+
if regscope_depth > matched_regid_depth or (
|
|
263
|
+
regscope_depth == matched_regid_depth
|
|
264
|
+
and regscope_start > matched_regid_start
|
|
265
|
+
):
|
|
152
266
|
matched_regid = reg.id
|
|
153
267
|
matched_regid_depth = regscope_depth
|
|
268
|
+
matched_regid_start = regscope_start
|
|
154
269
|
|
|
155
270
|
if matched_regid is None:
|
|
156
271
|
return None
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from codegen.models.types import AstID
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def move_ast_id(ast_id: AstID, source: AstID, target: AstID) -> AstID:
|
|
7
|
+
"""Move an AST to a new location"""
|
|
8
|
+
assert ast_id[: len(source)] == source
|
|
9
|
+
return target + ast_id[len(source) :]
|
|
@@ -3,27 +3,36 @@ from __future__ import annotations
|
|
|
3
3
|
from dataclasses import dataclass
|
|
4
4
|
from typing import TYPE_CHECKING, NamedTuple, Optional
|
|
5
5
|
|
|
6
|
-
from codegen.models.types import
|
|
6
|
+
from codegen.models.types import KEY, NO_KEY, AstID
|
|
7
|
+
from codegen.models.utils import move_ast_id
|
|
7
8
|
|
|
8
9
|
if TYPE_CHECKING:
|
|
9
10
|
from codegen.models.expr import ExprIdent
|
|
10
11
|
|
|
11
12
|
|
|
12
13
|
class VarScope(NamedTuple):
|
|
13
|
-
ast:
|
|
14
|
+
ast: AstID # the ast that has a child, which is an assign statement, that the first time the variable is defined
|
|
14
15
|
child_index_start: int # the index of the child (assign statement)
|
|
15
16
|
child_index_end: Optional[int] = (
|
|
16
17
|
None # the index of the child that the variable is deleted (exclusive)
|
|
17
18
|
)
|
|
18
19
|
|
|
19
20
|
@staticmethod
|
|
20
|
-
def from_ast_id(ast_id:
|
|
21
|
+
def from_ast_id(ast_id: AstID) -> VarScope:
|
|
21
22
|
"""Get the scope of a variable as if it is assigned to the given AST"""
|
|
22
23
|
return VarScope(ast_id[:-1], ast_id[-1], None)
|
|
23
24
|
|
|
24
25
|
def get_depth(self) -> int:
|
|
25
26
|
return len(self.ast) + 1
|
|
26
27
|
|
|
28
|
+
def move(self, source: AstID, target: AstID) -> VarScope:
|
|
29
|
+
"""Move the scope of the variable from an AST to a new AST location, assuming that the AST does not change, but only moves to a new location in another AST tree"""
|
|
30
|
+
return VarScope(
|
|
31
|
+
move_ast_id(self.ast, source, target),
|
|
32
|
+
self.child_index_start,
|
|
33
|
+
(self.child_index_end if self.child_index_end is not None else None),
|
|
34
|
+
)
|
|
35
|
+
|
|
27
36
|
|
|
28
37
|
@dataclass
|
|
29
38
|
class Var: # variable
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "codegen-2"
|
|
3
|
-
version = "2.16.
|
|
3
|
+
version = "2.16.2"
|
|
4
4
|
description = "Meta-programming"
|
|
5
5
|
authors = [{ name = "Binh Vu", email = "binh@toan2.com" }]
|
|
6
6
|
license = { text = "MIT" }
|
|
@@ -8,11 +8,14 @@ readme = "README.md"
|
|
|
8
8
|
requires-python = ">=3.11"
|
|
9
9
|
dependencies = []
|
|
10
10
|
|
|
11
|
-
[
|
|
12
|
-
dev = [
|
|
11
|
+
[dependency-groups]
|
|
12
|
+
dev = [
|
|
13
|
+
"pytest>=8.0.0",
|
|
14
|
+
"autoflake>=2.2.1",
|
|
15
|
+
]
|
|
13
16
|
|
|
14
17
|
[build-system]
|
|
15
|
-
requires = ["
|
|
18
|
+
requires = ["uv_build"]
|
|
16
19
|
build-backend = "uv_build"
|
|
17
20
|
|
|
18
21
|
[tool.uv.build-backend]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|