codegen-2 2.16.1__tar.gz → 2.16.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,14 +1,11 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: codegen-2
3
- Version: 2.16.1
3
+ Version: 2.16.2
4
4
  Summary: Meta-programming
5
5
  Author: Binh Vu
6
6
  Author-email: Binh Vu <binh@toan2.com>
7
7
  License: MIT
8
- Requires-Dist: pytest>=8.0.0 ; extra == 'dev'
9
- Requires-Dist: autoflake>=2.2.1 ; extra == 'dev'
10
8
  Requires-Python: >=3.11
11
- Provides-Extra: dev
12
9
  Description-Content-Type: text/markdown
13
10
 
14
11
  # codegen
@@ -3,7 +3,7 @@ import codegen.models.statement as stmt
3
3
  from codegen.models.ast import AST
4
4
  from codegen.models.expr import PredefinedFn
5
5
  from codegen.models.program import ImportHelper, Program
6
- from codegen.models.types import AST_ID, KEY
6
+ from codegen.models.types import KEY, AstID
7
7
  from codegen.models.var import DeferredVar, Var, VarScope
8
8
 
9
9
  __all__ = [
@@ -15,7 +15,7 @@ __all__ = [
15
15
  "Program",
16
16
  "stmt",
17
17
  "PredefinedFn",
18
- "AST_ID",
18
+ "AstID",
19
19
  "KEY",
20
20
  "ImportHelper",
21
21
  ]
@@ -1,5 +1,6 @@
1
1
  from __future__ import annotations
2
2
 
3
+ import copy
3
4
  from dataclasses import dataclass, field
4
5
  from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, Sequence
5
6
 
@@ -24,7 +25,8 @@ from codegen.models.statement import (
24
25
  Statement,
25
26
  TryStatement,
26
27
  )
27
- from codegen.models.types import AST_ID
28
+ from codegen.models.types import AstID
29
+ from codegen.models.utils import move_ast_id
28
30
  from codegen.models.var import DeferredVar, Var, VarScope
29
31
 
30
32
  if TYPE_CHECKING:
@@ -33,7 +35,7 @@ if TYPE_CHECKING:
33
35
 
34
36
  @dataclass
35
37
  class AST:
36
- id: AST_ID
38
+ id: AstID
37
39
  prog: Program
38
40
  stmt: Statement
39
41
  children: list[AST] = field(default_factory=list)
@@ -77,6 +79,24 @@ class AST:
77
79
  for child in self.children:
78
80
  child.freeze()
79
81
 
82
+ def clone(self, program: Program, var_memo: Optional[dict[int, Any]] = None) -> AST:
83
+ """Create a deep copy of this AST and all its children.
84
+
85
+ The cloned AST shares the same program reference and preserves
86
+ all IDs, but statements and children are independent copies that
87
+ can be modified without affecting the original.
88
+ """
89
+ ast = AST(self.id, program, copy.deepcopy(self.stmt, memo=var_memo))
90
+ ast._is_frozen = self._is_frozen
91
+ ast.children = [child.clone(program, var_memo) for child in self.children]
92
+ return ast
93
+
94
+ def move(self, source: AstID, target: AstID) -> AST:
95
+ self.id = move_ast_id(self.id, source, target)
96
+ for child in self.children:
97
+ child.move(source, target)
98
+ return self
99
+
80
100
  def import_(self, module: str, is_import_attr: bool, alias: Optional[str] = None):
81
101
  self._add_stmt(ImportStatement(module, is_import_attr, alias))
82
102
 
@@ -220,7 +240,7 @@ class AST:
220
240
  ast, context, stop = fn(ast, context)
221
241
  return ast
222
242
 
223
- def has_statement_between_ast(self, stmtcls: type[Statement], end_ast_id: AST_ID):
243
+ def has_statement_between_ast(self, stmtcls: type[Statement], end_ast_id: AstID):
224
244
  """Check if there is a statement of the given type the current AST and the end_ast (exclusive)"""
225
245
  for ast in self.children:
226
246
  for intermediate_ast in ast.find_ast_to(end_ast_id):
@@ -231,7 +251,7 @@ class AST:
231
251
  return True
232
252
  return False
233
253
 
234
- def find_ast_to(self, id: AST_ID):
254
+ def find_ast_to(self, id: AstID):
235
255
  """Iterate through ASTs that lead to the AST with the given id (inclusive)"""
236
256
  if self.id == id:
237
257
  yield self
@@ -242,7 +262,7 @@ class AST:
242
262
  yield from ast.find_ast_to(id)
243
263
  break
244
264
 
245
- def find_ast(self, id: AST_ID) -> Optional[AST]:
265
+ def find_ast(self, id: AstID) -> Optional[AST]:
246
266
  """Find the AST with the given id"""
247
267
  if len(self.id) > len(id):
248
268
  # the ast that we are looking for is not in the subtree of this ast
@@ -261,11 +281,11 @@ class AST:
261
281
  ast = ast.children[id[i]]
262
282
  return ast
263
283
 
264
- def next_child_id(self) -> AST_ID:
284
+ def next_child_id(self) -> AstID:
265
285
  """Get ID for the next child of this AST"""
266
286
  return self.id + (len(self.children),)
267
287
 
268
- def next_grandchild_id(self) -> AST_ID:
288
+ def next_grandchild_id(self) -> AstID:
269
289
  """Get ID for the next grandchild of this AST"""
270
290
  return self.id + (len(self.children), 0)
271
291
 
@@ -1,13 +1,13 @@
1
1
  from __future__ import annotations
2
2
 
3
- import ast
4
- from dataclasses import dataclass, field
5
- from typing import Optional
3
+ import copy
4
+ from dataclasses import dataclass, field, fields, is_dataclass
5
+ from typing import Any, Optional
6
6
 
7
7
  from codegen.models.ast import AST
8
8
  from codegen.models.expr import ExprIdent # You may need to adjust this import path
9
9
  from codegen.models.statement import BlockStatement, IfStatement
10
- from codegen.models.types import AST_ID, KEY
10
+ from codegen.models.types import KEY, AstID
11
11
  from codegen.models.var import Var, VarScope
12
12
 
13
13
 
@@ -29,13 +29,13 @@ class Program:
29
29
  self.import_area.import_(module, is_import_attr, alias)
30
30
  self.imported_modules.add(module)
31
31
 
32
- def get_ast_by_id(self, id: AST_ID) -> AST:
32
+ def get_ast_by_id(self, id: AstID) -> AST:
33
33
  ast = self.root
34
34
  for item in id:
35
35
  ast = ast.children[item]
36
36
  return ast
37
37
 
38
- def is_within_scope(self, ast: AST_ID, scope: VarScope) -> bool:
38
+ def is_within_scope(self, ast: AstID, scope: VarScope) -> bool:
39
39
  if len(ast) - 1 < len(scope.ast):
40
40
  return False
41
41
 
@@ -56,7 +56,7 @@ class Program:
56
56
  self,
57
57
  name: str,
58
58
  key: KEY,
59
- ast: AST_ID,
59
+ ast: AstID,
60
60
  force_name: Optional[str] = None,
61
61
  type: Optional[ExprIdent] = None,
62
62
  ) -> Var:
@@ -74,7 +74,7 @@ class Program:
74
74
  self,
75
75
  *,
76
76
  key: KEY,
77
- at: AST_ID,
77
+ at: AstID,
78
78
  ) -> Var:
79
79
  """De-reference a variable from the memory that is available at the provided AST"""
80
80
  reg = self.vars.find(key, at)
@@ -84,6 +84,82 @@ class Program:
84
84
  name=reg.name, key=key, register_id=reg.id, scope=reg.scope, type=reg.type
85
85
  )
86
86
 
87
+ def concatenate(self, *others: Program) -> Program:
88
+ """Return a new program containing this program followed by the others."""
89
+ program = Program()
90
+ program.extend(self, *others)
91
+ return program
92
+
93
+ def extend(self, *others: Program) -> None:
94
+ """Append other programs to this program."""
95
+ for source in others:
96
+ if source is self:
97
+ # Snapshot self-extension so we don't iterate source lists while appending to them.
98
+ source = source.concatenate()
99
+
100
+ body_child_offset = len(self.root.children) - 1
101
+ register_id_map = self.vars.extend(source.vars, body_child_offset)
102
+
103
+ var_memo: dict[int, Var] = {}
104
+ for var in self._collect_program_vars(source):
105
+ new_reg = register_id_map[var.register_id]
106
+ var_memo[id(var)] = Var(
107
+ name=var.name,
108
+ key=var.key,
109
+ register_id=new_reg.id,
110
+ scope=new_reg.scope,
111
+ force_name=var.force_name,
112
+ type=copy.deepcopy(var.type),
113
+ )
114
+
115
+ self._copy_children(source.import_area, self.import_area, var_memo)
116
+ for child in source.root.children:
117
+ if child is source.import_area:
118
+ continue
119
+ self.root.children.append(
120
+ child.clone(self, var_memo).move(child.id, self.root.next_child_id())
121
+ )
122
+ self.imported_modules.update(source.imported_modules)
123
+
124
+ def _copy_children(self, source_parent: AST, dest_parent: AST, var_memo: Optional[dict[int, Any]] = None):
125
+ for child in source_parent.children:
126
+ dest_parent.children.append(
127
+ child.clone(self, var_memo).move(source_parent.id, dest_parent.next_child_id())
128
+ )
129
+
130
+ @classmethod
131
+ def _collect_program_vars(cls, program: Program) -> list[Var]:
132
+ vars: list[Var] = []
133
+ cls._collect_ast_vars(program.root, vars)
134
+ return vars
135
+
136
+ @classmethod
137
+ def _collect_ast_vars(cls, ast: AST, vars: list[Var]):
138
+ cls._collect_vars(ast.stmt, vars)
139
+ for child in ast.children:
140
+ cls._collect_ast_vars(child, vars)
141
+
142
+ @classmethod
143
+ def _collect_vars(cls, obj: Any, vars: list[Var]):
144
+ if isinstance(obj, Var):
145
+ vars.append(obj)
146
+ return
147
+
148
+ if is_dataclass(obj) and not isinstance(obj, type):
149
+ for field in fields(obj):
150
+ cls._collect_vars(getattr(obj, field.name), vars)
151
+ return
152
+
153
+ if isinstance(obj, dict):
154
+ for key, value in obj.items():
155
+ cls._collect_vars(key, vars)
156
+ cls._collect_vars(value, vars)
157
+ return
158
+
159
+ if isinstance(obj, (list, tuple, set, frozenset)):
160
+ for item in obj:
161
+ cls._collect_vars(item, vars)
162
+
87
163
 
88
164
  @dataclass
89
165
  class VarRegister:
@@ -101,11 +177,43 @@ class VarRegisters:
101
177
  registers: list[VarRegister] = field(default_factory=list)
102
178
  key2registers: dict[KEY, list[int]] = field(default_factory=dict)
103
179
 
180
+ def extend(
181
+ self, source: VarRegisters, body_child_offset: int
182
+ ) -> dict[int, VarRegister]:
183
+ register_id_map: dict[int, VarRegister] = {}
184
+
185
+ for reg in source.registers:
186
+ if len(reg.scope.ast) == 0:
187
+ new_scope = VarScope(
188
+ ast=(),
189
+ child_index_start=reg.scope.child_index_start + body_child_offset,
190
+ child_index_end=(reg.scope.child_index_end + body_child_offset) if reg.scope.child_index_end is not None else None
191
+ )
192
+ else:
193
+ new_scope = reg.scope.move(
194
+ (reg.scope.ast[0],),
195
+ (reg.scope.ast[0] + body_child_offset,)
196
+ )
197
+
198
+ new_reg = VarRegister(
199
+ id=len(self.registers),
200
+ name=reg.name,
201
+ key=reg.key,
202
+ scope=new_scope,
203
+ force_name=reg.force_name,
204
+ type=copy.deepcopy(reg.type),
205
+ )
206
+ self.registers.append(new_reg)
207
+ self.key2registers.setdefault(new_reg.key, []).append(new_reg.id)
208
+ register_id_map[reg.id] = new_reg
209
+
210
+ return register_id_map
211
+
104
212
  def register(
105
213
  self,
106
214
  name: str,
107
215
  key: KEY,
108
- ast: AST_ID,
216
+ ast: AstID,
109
217
  force_name: Optional[str] = None,
110
218
  type: Optional[ExprIdent] = None,
111
219
  ) -> VarRegister:
@@ -135,22 +243,29 @@ class VarRegisters:
135
243
  self.key2registers.setdefault(key, []).append(reg.id)
136
244
  return reg
137
245
 
138
- def find(self, key: KEY, ast: AST_ID) -> Optional[VarRegister]:
246
+ def find(self, key: KEY, ast: AstID) -> Optional[VarRegister]:
139
247
  """Find the most specific register by name, key that is available in the given ast. If
140
- there are multiple matches, the most specific register is the one with the largest depth.
248
+ there are multiple matches, the most specific register is the one with the largest depth,
249
+ then the latest start point.
141
250
  """
142
251
  if key not in self.key2registers:
143
252
  return None
144
253
 
145
254
  matched_regid = None
146
255
  matched_regid_depth = -1
256
+ matched_regid_start: AstID = tuple()
147
257
  for regid in self.key2registers[key]:
148
258
  reg = self.registers[regid]
149
259
  if self.program.is_within_scope(ast, reg.scope):
150
260
  regscope_depth = reg.scope.get_depth()
151
- if regscope_depth > matched_regid_depth:
261
+ regscope_start = reg.scope.ast + (reg.scope.child_index_start,)
262
+ if regscope_depth > matched_regid_depth or (
263
+ regscope_depth == matched_regid_depth
264
+ and regscope_start > matched_regid_start
265
+ ):
152
266
  matched_regid = reg.id
153
267
  matched_regid_depth = regscope_depth
268
+ matched_regid_start = regscope_start
154
269
 
155
270
  if matched_regid is None:
156
271
  return None
@@ -1,9 +1,9 @@
1
1
  from __future__ import annotations
2
2
 
3
- from typing import Literal, TypeAlias
3
+ from typing import Literal
4
4
 
5
5
  KEY = str | tuple
6
6
  NO_KEY = tuple()
7
7
  REGISTER_TYPE = Literal["var"]
8
8
 
9
- AST_ID: TypeAlias = tuple[int, ...]
9
+ AstID = tuple[int, ...]
@@ -0,0 +1,9 @@
1
+ from __future__ import annotations
2
+
3
+ from codegen.models.types import AstID
4
+
5
+
6
+ def move_ast_id(ast_id: AstID, source: AstID, target: AstID) -> AstID:
7
+ """Move an AST to a new location"""
8
+ assert ast_id[: len(source)] == source
9
+ return target + ast_id[len(source) :]
@@ -3,27 +3,36 @@ from __future__ import annotations
3
3
  from dataclasses import dataclass
4
4
  from typing import TYPE_CHECKING, NamedTuple, Optional
5
5
 
6
- from codegen.models.types import AST_ID, KEY, NO_KEY
6
+ from codegen.models.types import KEY, NO_KEY, AstID
7
+ from codegen.models.utils import move_ast_id
7
8
 
8
9
  if TYPE_CHECKING:
9
10
  from codegen.models.expr import ExprIdent
10
11
 
11
12
 
12
13
  class VarScope(NamedTuple):
13
- ast: AST_ID # the ast that has a child, which is an assign statement, that the first time the variable is defined
14
+ ast: AstID # the ast that has a child, which is an assign statement, that the first time the variable is defined
14
15
  child_index_start: int # the index of the child (assign statement)
15
16
  child_index_end: Optional[int] = (
16
17
  None # the index of the child that the variable is deleted (exclusive)
17
18
  )
18
19
 
19
20
  @staticmethod
20
- def from_ast_id(ast_id: AST_ID) -> VarScope:
21
+ def from_ast_id(ast_id: AstID) -> VarScope:
21
22
  """Get the scope of a variable as if it is assigned to the given AST"""
22
23
  return VarScope(ast_id[:-1], ast_id[-1], None)
23
24
 
24
25
  def get_depth(self) -> int:
25
26
  return len(self.ast) + 1
26
27
 
28
+ def move(self, source: AstID, target: AstID) -> VarScope:
29
+ """Move the scope of the variable from an AST to a new AST location, assuming that the AST does not change, but only moves to a new location in another AST tree"""
30
+ return VarScope(
31
+ move_ast_id(self.ast, source, target),
32
+ self.child_index_start,
33
+ (self.child_index_end if self.child_index_end is not None else None),
34
+ )
35
+
27
36
 
28
37
  @dataclass
29
38
  class Var: # variable
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "codegen-2"
3
- version = "2.16.1"
3
+ version = "2.16.2"
4
4
  description = "Meta-programming"
5
5
  authors = [{ name = "Binh Vu", email = "binh@toan2.com" }]
6
6
  license = { text = "MIT" }
@@ -8,11 +8,14 @@ readme = "README.md"
8
8
  requires-python = ">=3.11"
9
9
  dependencies = []
10
10
 
11
- [project.optional-dependencies]
12
- dev = ["pytest>=8.0.0", "autoflake>=2.2.1"]
11
+ [dependency-groups]
12
+ dev = [
13
+ "pytest>=8.0.0",
14
+ "autoflake>=2.2.1",
15
+ ]
13
16
 
14
17
  [build-system]
15
- requires = ["uv-build"]
18
+ requires = ["uv_build"]
16
19
  build-backend = "uv_build"
17
20
 
18
21
  [tool.uv.build-backend]
File without changes