codegen-2 1.4.0__tar.gz → 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {codegen_2-1.4.0 → codegen_2-2.0.0}/PKG-INFO +1 -2
- {codegen_2-1.4.0 → codegen_2-2.0.0}/codegen/models/__init__.py +3 -4
- {codegen_2-1.4.0 → codegen_2-2.0.0}/codegen/models/ast.py +86 -51
- {codegen_2-1.4.0 → codegen_2-2.0.0}/codegen/models/expr.py +10 -1
- codegen_2-2.0.0/codegen/models/program.py +137 -0
- {codegen_2-1.4.0 → codegen_2-2.0.0}/codegen/models/statement.py +15 -14
- codegen_2-2.0.0/codegen/models/types.py +9 -0
- codegen_2-2.0.0/codegen/models/var.py +61 -0
- {codegen_2-1.4.0 → codegen_2-2.0.0}/pyproject.toml +1 -2
- codegen_2-1.4.0/codegen/models/memory.py +0 -199
- codegen_2-1.4.0/codegen/models/program.py +0 -38
- codegen_2-1.4.0/codegen/models/types.py +0 -18
- {codegen_2-1.4.0 → codegen_2-2.0.0}/LICENSE +0 -0
- {codegen_2-1.4.0 → codegen_2-2.0.0}/README.md +0 -0
- {codegen_2-1.4.0 → codegen_2-2.0.0}/codegen/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: codegen-2
|
|
3
|
-
Version:
|
|
3
|
+
Version: 2.0.0
|
|
4
4
|
Summary: Meta-programming
|
|
5
5
|
License: MIT
|
|
6
6
|
Author: Binh Vu
|
|
@@ -10,7 +10,6 @@ Classifier: License :: OSI Approved :: MIT License
|
|
|
10
10
|
Classifier: Programming Language :: Python :: 3
|
|
11
11
|
Classifier: Programming Language :: Python :: 3.11
|
|
12
12
|
Classifier: Programming Language :: Python :: 3.12
|
|
13
|
-
Requires-Dist: pandas (>=2.2.0,<3.0.0)
|
|
14
13
|
Description-Content-Type: text/markdown
|
|
15
14
|
|
|
16
15
|
# codegen
|
|
@@ -2,20 +2,19 @@ import codegen.models.expr as expr
|
|
|
2
2
|
import codegen.models.statement as stmt
|
|
3
3
|
from codegen.models.ast import AST
|
|
4
4
|
from codegen.models.expr import PredefinedFn
|
|
5
|
-
from codegen.models.memory import Memory, Var, VarScope
|
|
6
5
|
from codegen.models.program import Program
|
|
7
|
-
from codegen.models.types import AST_ID, KEY
|
|
6
|
+
from codegen.models.types import AST_ID, KEY
|
|
7
|
+
from codegen.models.var import DeferredVar, Var, VarScope
|
|
8
8
|
|
|
9
9
|
__all__ = [
|
|
10
10
|
"expr",
|
|
11
11
|
"AST",
|
|
12
|
-
"Memory",
|
|
13
12
|
"Var",
|
|
14
13
|
"VarScope",
|
|
14
|
+
"DeferredVar",
|
|
15
15
|
"Program",
|
|
16
16
|
"stmt",
|
|
17
17
|
"PredefinedFn",
|
|
18
18
|
"AST_ID",
|
|
19
19
|
"KEY",
|
|
20
|
-
"SCOPE",
|
|
21
20
|
]
|
|
@@ -1,15 +1,13 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
from dataclasses import dataclass, field
|
|
4
|
-
from typing import Any, Callable, Optional
|
|
4
|
+
from typing import TYPE_CHECKING, Any, Callable, Optional
|
|
5
5
|
|
|
6
6
|
from codegen.models.expr import ExceptionExpr, Expr
|
|
7
|
-
from codegen.models.memory import Memory, Var, VarScope
|
|
8
7
|
from codegen.models.statement import (
|
|
9
8
|
AssignStatement,
|
|
10
9
|
BlockStatement,
|
|
11
10
|
Comment,
|
|
12
|
-
ContinueStatement,
|
|
13
11
|
DefFuncStatement,
|
|
14
12
|
ElseStatement,
|
|
15
13
|
ExceptionStatement,
|
|
@@ -17,17 +15,22 @@ from codegen.models.statement import (
|
|
|
17
15
|
IfStatement,
|
|
18
16
|
ImportStatement,
|
|
19
17
|
LineBreak,
|
|
20
|
-
|
|
18
|
+
PythonStatement,
|
|
21
19
|
ReturnStatement,
|
|
22
20
|
SingleExprStatement,
|
|
23
21
|
Statement,
|
|
24
22
|
)
|
|
25
23
|
from codegen.models.types import AST_ID
|
|
24
|
+
from codegen.models.var import DeferredVar, Var, VarScope
|
|
25
|
+
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
from codegen.models.program import Program
|
|
26
28
|
|
|
27
29
|
|
|
28
30
|
@dataclass
|
|
29
31
|
class AST:
|
|
30
32
|
id: AST_ID
|
|
33
|
+
prog: Program
|
|
31
34
|
stmt: Statement
|
|
32
35
|
children: list[AST] = field(default_factory=list)
|
|
33
36
|
_is_frozen: bool = (
|
|
@@ -35,13 +38,15 @@ class AST:
|
|
|
35
38
|
)
|
|
36
39
|
|
|
37
40
|
@staticmethod
|
|
38
|
-
def root():
|
|
39
|
-
return AST(
|
|
41
|
+
def root(prog: Program):
|
|
42
|
+
return AST(tuple(), prog, BlockStatement())
|
|
40
43
|
|
|
41
44
|
def __call__(
|
|
42
45
|
self, *args: Callable[[AST], Any] | Statement, return_self: bool = False
|
|
43
46
|
) -> Optional[AST]:
|
|
44
47
|
"""Allow to build the graph hierarchically"""
|
|
48
|
+
assert isinstance(return_self, bool)
|
|
49
|
+
|
|
45
50
|
return_val = None
|
|
46
51
|
|
|
47
52
|
for fn in args:
|
|
@@ -61,8 +66,8 @@ class AST:
|
|
|
61
66
|
for child in self.children:
|
|
62
67
|
child.freeze()
|
|
63
68
|
|
|
64
|
-
def import_(self, module: str):
|
|
65
|
-
self._add_stmt(ImportStatement(module))
|
|
69
|
+
def import_(self, module: str, is_import_attr: bool):
|
|
70
|
+
self._add_stmt(ImportStatement(module, is_import_attr))
|
|
66
71
|
|
|
67
72
|
def return_(self, expr: Expr):
|
|
68
73
|
self._add_stmt(ReturnStatement(expr))
|
|
@@ -70,11 +75,20 @@ class AST:
|
|
|
70
75
|
def linebreak(self):
|
|
71
76
|
self._add_stmt(LineBreak())
|
|
72
77
|
|
|
78
|
+
def block(self):
|
|
79
|
+
return self._add_stmt(BlockStatement())
|
|
80
|
+
|
|
73
81
|
def comment(self, comment: str):
|
|
74
82
|
self._add_stmt(Comment(comment))
|
|
75
83
|
|
|
76
|
-
def func(self, name: str, vars: list[
|
|
77
|
-
|
|
84
|
+
def func(self, name: str, vars: list[DeferredVar]):
|
|
85
|
+
"""Define a function. The input variables are deferred vars as they are created for this function (i.e., must not be predefined prior to this function)"""
|
|
86
|
+
grandchild_id = self.next_grandchild_id()
|
|
87
|
+
for var in vars:
|
|
88
|
+
var.set_var(
|
|
89
|
+
self.prog.create_var(var.name, var.key, grandchild_id, var.force_name)
|
|
90
|
+
)
|
|
91
|
+
return self._add_stmt(DefFuncStatement(name, [var.get_var() for var in vars]))
|
|
78
92
|
|
|
79
93
|
def expr(self, expr: Expr):
|
|
80
94
|
return self._add_stmt(SingleExprStatement(expr))
|
|
@@ -82,17 +96,33 @@ class AST:
|
|
|
82
96
|
def raise_exception(self, expr: ExceptionExpr):
|
|
83
97
|
return self._add_stmt(ExceptionStatement(expr))
|
|
84
98
|
|
|
85
|
-
def assign(self,
|
|
86
|
-
|
|
99
|
+
def assign(self, var: DeferredVar | Var, expr: Expr):
|
|
100
|
+
"""When we assign the variable, we accept that either you create/declare the variable for the first time or reassign it"""
|
|
101
|
+
if isinstance(var, DeferredVar):
|
|
102
|
+
# create the variable for the first time
|
|
103
|
+
real_var = self.prog.create_var(
|
|
104
|
+
var.name, var.key, self.next_child_id(), var.force_name
|
|
105
|
+
)
|
|
106
|
+
var.set_var(real_var)
|
|
107
|
+
var = real_var
|
|
108
|
+
else:
|
|
109
|
+
# the variable is already created -- and we are just reassigning it
|
|
110
|
+
# we have to make sure that it's accessible from the current scope
|
|
111
|
+
assert self.prog.is_within_scope(self.next_child_id(), var.scope)
|
|
112
|
+
|
|
87
113
|
self._add_stmt(AssignStatement(var, expr))
|
|
88
|
-
var.set_scope(mem, scope)
|
|
89
114
|
|
|
90
|
-
def for_loop(self,
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
115
|
+
def for_loop(self, item: DeferredVar, iter: Expr):
|
|
116
|
+
"""When we construct a for-loop, the item (or var) is always declared. We don't allow reassigning the variable in the for-loop."""
|
|
117
|
+
assert (
|
|
118
|
+
isinstance(item, DeferredVar) and item.has_not_been_created()
|
|
119
|
+
), "The item must be a new variable"
|
|
120
|
+
var = self.prog.create_var(
|
|
121
|
+
item.name, item.key, self.next_grandchild_id(), item.force_name
|
|
122
|
+
)
|
|
123
|
+
item.set_var(var)
|
|
124
|
+
|
|
125
|
+
return self._add_stmt(ForLoopStatement(var, iter))
|
|
96
126
|
|
|
97
127
|
def if_(self, condition: Expr):
|
|
98
128
|
return self._add_stmt(IfStatement(condition))
|
|
@@ -101,6 +131,9 @@ class AST:
|
|
|
101
131
|
assert len(self.children) > 0 and isinstance(self.children[-1].stmt, Statement)
|
|
102
132
|
return self._add_stmt(ElseStatement())
|
|
103
133
|
|
|
134
|
+
def python_stmt(self, stmt: str) -> AST:
|
|
135
|
+
return self._add_stmt(PythonStatement(stmt))
|
|
136
|
+
|
|
104
137
|
def update_recursively(
|
|
105
138
|
self, fn: Callable[[AST, Any], tuple[AST, Any, bool]], context: Any
|
|
106
139
|
):
|
|
@@ -113,31 +146,15 @@ class AST:
|
|
|
113
146
|
ast, context, stop = fn(ast, context)
|
|
114
147
|
return ast
|
|
115
148
|
|
|
116
|
-
def to_python(self, level: int =
|
|
149
|
+
def to_python(self, level: int = 0):
|
|
117
150
|
"""Convert the AST to python code"""
|
|
118
|
-
if level == -1:
|
|
119
|
-
assert self.id == "root"
|
|
120
|
-
return "".join(
|
|
121
|
-
[
|
|
122
|
-
child.to_python(level + 1)
|
|
123
|
-
+ ("\n" if ci < len(self.children) - 1 else "")
|
|
124
|
-
for ci, child in enumerate(self.children)
|
|
125
|
-
]
|
|
126
|
-
)
|
|
127
|
-
|
|
128
151
|
if isinstance(self.stmt, BlockStatement):
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
prog = ["\t" * level + self.stmt.to_python()]
|
|
132
|
-
|
|
133
|
-
for child in self.children:
|
|
134
|
-
prog.append("\n")
|
|
135
|
-
prog.append(child.to_python(level + 1))
|
|
136
|
-
return "".join(prog)
|
|
152
|
+
# there is no direct statement and the level does not increase after this statement
|
|
153
|
+
return "\n".join([child.to_python(level) for child in self.children])
|
|
137
154
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
return
|
|
155
|
+
prog = ["\t" * level + self.stmt.to_python()]
|
|
156
|
+
prog.extend((child.to_python(level + 1) for child in self.children))
|
|
157
|
+
return "\n".join(prog)
|
|
141
158
|
|
|
142
159
|
def has_statement_between_ast(self, stmtcls: type[Statement], end_ast_id: AST_ID):
|
|
143
160
|
"""Check if there is a statement of the given type the current AST and the end_ast (exclusive)"""
|
|
@@ -162,20 +179,38 @@ class AST:
|
|
|
162
179
|
|
|
163
180
|
def find_ast(self, id: AST_ID) -> Optional[AST]:
|
|
164
181
|
"""Find the AST with the given id"""
|
|
165
|
-
if self.id
|
|
182
|
+
if len(self.id) > len(id):
|
|
183
|
+
# the ast that we are looking for is not in the subtree of this ast
|
|
184
|
+
return None
|
|
185
|
+
|
|
186
|
+
for i in range(len(self.id)):
|
|
187
|
+
if self.id[i] != id[i]:
|
|
188
|
+
# the ast that we are looking for is not in the subtree of this ast
|
|
189
|
+
return None
|
|
190
|
+
|
|
191
|
+
if len(self.id) == len(id):
|
|
166
192
|
return self
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
return
|
|
193
|
+
|
|
194
|
+
ast = self
|
|
195
|
+
for i in range(len(self.id), len(id)):
|
|
196
|
+
ast = ast.children[id[i]]
|
|
197
|
+
return ast
|
|
198
|
+
|
|
199
|
+
def next_child_id(self) -> AST_ID:
|
|
200
|
+
"""Get ID for the next child of this AST"""
|
|
201
|
+
return self.id + (len(self.children),)
|
|
202
|
+
|
|
203
|
+
def next_grandchild_id(self) -> AST_ID:
|
|
204
|
+
"""Get ID for the next grandchild of this AST"""
|
|
205
|
+
return self.id + (len(self.children), 0)
|
|
206
|
+
|
|
207
|
+
def next_var_scope(self) -> VarScope:
|
|
208
|
+
"""Get a scope for the next variable that will be have if it is assigned to this AST"""
|
|
209
|
+
return VarScope(self.id, len(self.children))
|
|
172
210
|
|
|
173
211
|
def _add_stmt(self, stmt: Statement):
|
|
174
212
|
if self._is_frozen:
|
|
175
213
|
raise Exception("The AST is frozen and cannot be modified")
|
|
176
|
-
ast = AST(self.
|
|
214
|
+
ast = AST(self.next_child_id(), self.prog, stmt)
|
|
177
215
|
self.children.append(ast)
|
|
178
216
|
return ast
|
|
179
|
-
|
|
180
|
-
def _next_child_id(self):
|
|
181
|
-
return self.id + "__" + str(len(self.children))
|
|
@@ -5,7 +5,7 @@ from abc import ABC, abstractmethod
|
|
|
5
5
|
from dataclasses import dataclass
|
|
6
6
|
from typing import Any, Optional, Sequence
|
|
7
7
|
|
|
8
|
-
from codegen.models.
|
|
8
|
+
from codegen.models.var import Var
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
class Expr(ABC):
|
|
@@ -140,6 +140,15 @@ class PredefinedFn:
|
|
|
140
140
|
def to_python(self):
|
|
141
141
|
return f"{self.collection.to_python()}[{self.item.to_python()}]"
|
|
142
142
|
|
|
143
|
+
@dataclass
|
|
144
|
+
class item_setter(Expr):
|
|
145
|
+
collection: Expr
|
|
146
|
+
item: Expr
|
|
147
|
+
value: Expr
|
|
148
|
+
|
|
149
|
+
def to_python(self):
|
|
150
|
+
return f"{self.collection.to_python()}[{self.item.to_python()}] = {self.value.to_python()}"
|
|
151
|
+
|
|
143
152
|
@dataclass
|
|
144
153
|
class len(Expr):
|
|
145
154
|
collection: Expr
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import ast
|
|
4
|
+
from dataclasses import dataclass, field
|
|
5
|
+
from typing import Optional
|
|
6
|
+
|
|
7
|
+
from codegen.models.ast import AST
|
|
8
|
+
from codegen.models.statement import BlockStatement
|
|
9
|
+
from codegen.models.types import AST_ID, KEY
|
|
10
|
+
from codegen.models.var import Var, VarScope
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
@dataclass(init=False)
|
|
14
|
+
class Program:
|
|
15
|
+
root: AST
|
|
16
|
+
vars: VarRegisters
|
|
17
|
+
import_area: AST
|
|
18
|
+
|
|
19
|
+
def __init__(self):
|
|
20
|
+
self.vars = VarRegisters(self)
|
|
21
|
+
self.root = AST.root(self)
|
|
22
|
+
self.import_area = self.root._add_stmt(BlockStatement(has_owned_env=False))
|
|
23
|
+
|
|
24
|
+
def import_(self, module: str, is_import_attr: bool):
|
|
25
|
+
self.import_area.import_(module, is_import_attr)
|
|
26
|
+
|
|
27
|
+
def get_ast_by_id(self, id: AST_ID) -> AST:
|
|
28
|
+
ast = self.root
|
|
29
|
+
for item in id:
|
|
30
|
+
ast = ast.children[item]
|
|
31
|
+
return ast
|
|
32
|
+
|
|
33
|
+
def is_within_scope(self, ast: AST_ID, scope: VarScope) -> bool:
|
|
34
|
+
if len(ast) - 1 < len(scope.ast):
|
|
35
|
+
return False
|
|
36
|
+
|
|
37
|
+
for i in range(len(scope.ast)):
|
|
38
|
+
if ast[i] != scope.ast[i]:
|
|
39
|
+
return False
|
|
40
|
+
|
|
41
|
+
if len(ast) - 1 == len(scope.ast):
|
|
42
|
+
return ast[-1] >= scope.child_index_start and (
|
|
43
|
+
scope.child_index_end is None or ast[-1] < scope.child_index_end
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
return ast[len(scope.ast)] >= scope.child_index_start and (
|
|
47
|
+
scope.child_index_end is None or ast[len(scope.ast)] < scope.child_index_end
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
def create_var(
|
|
51
|
+
self, name: str, key: KEY, ast: AST_ID, force_name: Optional[str] = None
|
|
52
|
+
) -> Var:
|
|
53
|
+
reg = self.vars.register(name, key, ast, force_name)
|
|
54
|
+
return Var(
|
|
55
|
+
name=name,
|
|
56
|
+
key=key,
|
|
57
|
+
register_id=reg.id,
|
|
58
|
+
scope=reg.scope,
|
|
59
|
+
force_name=force_name,
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
def get_var(
|
|
63
|
+
self,
|
|
64
|
+
*,
|
|
65
|
+
key: KEY,
|
|
66
|
+
at: AST_ID,
|
|
67
|
+
) -> Var:
|
|
68
|
+
"""De-reference a variable from the memory that is available at the provided AST"""
|
|
69
|
+
reg = self.vars.find(key, at)
|
|
70
|
+
if reg is None:
|
|
71
|
+
raise KeyError(f"Variable with key {key} is not found in the current scope")
|
|
72
|
+
return Var(reg.name, key, reg.id, reg.scope)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@dataclass
|
|
76
|
+
class VarRegister:
|
|
77
|
+
id: int
|
|
78
|
+
name: str
|
|
79
|
+
key: KEY
|
|
80
|
+
scope: VarScope
|
|
81
|
+
force_name: Optional[str] = None
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@dataclass
|
|
85
|
+
class VarRegisters:
|
|
86
|
+
program: Program
|
|
87
|
+
registers: list[VarRegister] = field(default_factory=list)
|
|
88
|
+
key2registers: dict[KEY, list[int]] = field(default_factory=dict)
|
|
89
|
+
|
|
90
|
+
def register(
|
|
91
|
+
self, name: str, key: KEY, ast: AST_ID, force_name: Optional[str] = None
|
|
92
|
+
) -> VarRegister:
|
|
93
|
+
"""Register a new variable with the given name, key, then return the register id.
|
|
94
|
+
|
|
95
|
+
The scope of the variable is created from the given ast.
|
|
96
|
+
"""
|
|
97
|
+
reg = self.find(key, ast)
|
|
98
|
+
if reg is not None:
|
|
99
|
+
# we found an existing register -- we need to test if we can override the existing register, which
|
|
100
|
+
# means if we go out of the new register scope, the value of the existing register is restored.
|
|
101
|
+
|
|
102
|
+
# for now, we don't support overriding an existing register
|
|
103
|
+
raise ValueError(
|
|
104
|
+
f"Variable {name} with key {key} is already registered in the current scope"
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
reg = VarRegister(
|
|
108
|
+
id=len(self.registers),
|
|
109
|
+
name=name,
|
|
110
|
+
key=key,
|
|
111
|
+
scope=VarScope.from_ast_id(ast),
|
|
112
|
+
force_name=force_name,
|
|
113
|
+
)
|
|
114
|
+
self.registers.append(reg)
|
|
115
|
+
self.key2registers.setdefault(key, []).append(reg.id)
|
|
116
|
+
return reg
|
|
117
|
+
|
|
118
|
+
def find(self, key: KEY, ast: AST_ID) -> Optional[VarRegister]:
|
|
119
|
+
"""Find the most specific register by name, key that is available in the given ast. If
|
|
120
|
+
there are multiple matches, the most specific register is the one with the largest depth.
|
|
121
|
+
"""
|
|
122
|
+
if key not in self.key2registers:
|
|
123
|
+
return None
|
|
124
|
+
|
|
125
|
+
matched_regid = None
|
|
126
|
+
matched_regid_depth = -1
|
|
127
|
+
for regid in self.key2registers[key]:
|
|
128
|
+
reg = self.registers[regid]
|
|
129
|
+
if self.program.is_within_scope(ast, reg.scope):
|
|
130
|
+
regscope_depth = reg.scope.get_depth()
|
|
131
|
+
if regscope_depth > matched_regid_depth:
|
|
132
|
+
matched_regid = reg.id
|
|
133
|
+
matched_regid_depth = regscope_depth
|
|
134
|
+
|
|
135
|
+
if matched_regid is None:
|
|
136
|
+
return None
|
|
137
|
+
return self.registers[matched_regid]
|
|
@@ -4,7 +4,7 @@ from abc import ABC, abstractmethod
|
|
|
4
4
|
from dataclasses import dataclass, field
|
|
5
5
|
|
|
6
6
|
from codegen.models.expr import ExceptionExpr, Expr
|
|
7
|
-
from codegen.models.
|
|
7
|
+
from codegen.models.var import Var
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
class Statement(ABC):
|
|
@@ -14,23 +14,15 @@ class Statement(ABC):
|
|
|
14
14
|
raise NotImplementedError()
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
class NoStatement(Statement):
|
|
18
|
-
def __repr__(self):
|
|
19
|
-
return "NoStatement()"
|
|
20
|
-
|
|
21
|
-
def to_python(self):
|
|
22
|
-
return "pass"
|
|
23
|
-
|
|
24
|
-
|
|
25
17
|
@dataclass
|
|
26
18
|
class BlockStatement(Statement):
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
19
|
+
# whether the block has its own environment or not -- meaning any variables declared inside the block will be
|
|
20
|
+
# only visible inside the block
|
|
21
|
+
has_owned_env: bool = True
|
|
30
22
|
|
|
31
23
|
def to_python(self):
|
|
32
24
|
raise Exception(
|
|
33
|
-
"BlockStatement
|
|
25
|
+
"BlockStatement doesn't have any direct statement. You can use it to create scope for variables."
|
|
34
26
|
)
|
|
35
27
|
|
|
36
28
|
|
|
@@ -42,9 +34,10 @@ class LineBreak(Statement):
|
|
|
42
34
|
@dataclass
|
|
43
35
|
class ImportStatement(Statement):
|
|
44
36
|
module: str
|
|
37
|
+
is_import_attr: bool
|
|
45
38
|
|
|
46
39
|
def to_python(self):
|
|
47
|
-
if self.module.find(".") != -1:
|
|
40
|
+
if self.module.find(".") != -1 and self.is_import_attr:
|
|
48
41
|
module, attr = self.module.rsplit(".", 1)
|
|
49
42
|
return f"from {module} import {attr}"
|
|
50
43
|
return f"import {self.module}"
|
|
@@ -133,3 +126,11 @@ class Comment(Statement):
|
|
|
133
126
|
|
|
134
127
|
def to_python(self):
|
|
135
128
|
return f"# {self.comment}"
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
@dataclass
|
|
132
|
+
class PythonStatement(Statement):
|
|
133
|
+
stmt: str
|
|
134
|
+
|
|
135
|
+
def to_python(self):
|
|
136
|
+
return self.stmt
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import NamedTuple, Optional
|
|
5
|
+
|
|
6
|
+
from codegen.models.types import AST_ID, KEY, NO_KEY
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class VarScope(NamedTuple):
|
|
10
|
+
ast: AST_ID # the ast that has a child, which is an assign statement, that the first time the variable is defined
|
|
11
|
+
child_index_start: int # the index of the child (assign statement)
|
|
12
|
+
child_index_end: Optional[int] = (
|
|
13
|
+
None # the index of the child that the variable is deleted (exclusive)
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
@staticmethod
|
|
17
|
+
def from_ast_id(ast_id: AST_ID) -> VarScope:
|
|
18
|
+
"""Get the scope of a variable as if it is assigned to the given AST"""
|
|
19
|
+
return VarScope(ast_id[:-1], ast_id[-1], None)
|
|
20
|
+
|
|
21
|
+
def get_depth(self) -> int:
|
|
22
|
+
return len(self.ast) + 1
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class Var: # variable
|
|
27
|
+
name: str
|
|
28
|
+
key: KEY
|
|
29
|
+
register_id: int
|
|
30
|
+
scope: VarScope
|
|
31
|
+
force_name: Optional[str] = None
|
|
32
|
+
|
|
33
|
+
def get_name(self) -> str:
|
|
34
|
+
if self.force_name is None:
|
|
35
|
+
return f"{self.name}_{self.register_id}"
|
|
36
|
+
return self.force_name
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass
|
|
40
|
+
class DeferredVar:
|
|
41
|
+
"""Containings the information to create a variable"""
|
|
42
|
+
|
|
43
|
+
name: str
|
|
44
|
+
key: KEY = NO_KEY
|
|
45
|
+
force_name: Optional[str] = None
|
|
46
|
+
|
|
47
|
+
# the variable that we want to create --- this will be not None when the variable is actually created
|
|
48
|
+
_var: Optional[Var] = None
|
|
49
|
+
|
|
50
|
+
def get_var(self) -> Var:
|
|
51
|
+
if self._var is None:
|
|
52
|
+
raise ValueError("The variable has not been created yet")
|
|
53
|
+
return self._var
|
|
54
|
+
|
|
55
|
+
def set_var(self, var: Var):
|
|
56
|
+
if self._var is not None:
|
|
57
|
+
raise ValueError("The variable has been created already")
|
|
58
|
+
self._var = var
|
|
59
|
+
|
|
60
|
+
def has_not_been_created(self) -> bool:
|
|
61
|
+
return self._var is None
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "codegen-2"
|
|
3
|
-
version = "
|
|
3
|
+
version = "2.0.0"
|
|
4
4
|
description = "Meta-programming"
|
|
5
5
|
authors = ["Binh Vu <binh@toan2.com>"]
|
|
6
6
|
license = "MIT"
|
|
@@ -9,7 +9,6 @@ packages = [{ include = "codegen" }]
|
|
|
9
9
|
|
|
10
10
|
[tool.poetry.dependencies]
|
|
11
11
|
python = "^3.11"
|
|
12
|
-
pandas = "^2.2.0"
|
|
13
12
|
|
|
14
13
|
[tool.poetry.group.dev.dependencies]
|
|
15
14
|
black = "^24.1.1"
|
|
@@ -1,199 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from dataclasses import dataclass, field
|
|
4
|
-
from typing import Literal, NamedTuple, Optional, cast
|
|
5
|
-
|
|
6
|
-
import pandas as pd
|
|
7
|
-
from codegen.models.types import (
|
|
8
|
-
AST_ID,
|
|
9
|
-
KEY,
|
|
10
|
-
NO_AST_ID,
|
|
11
|
-
NO_KEY,
|
|
12
|
-
NO_SCOPE,
|
|
13
|
-
REGISTER_TYPE,
|
|
14
|
-
SCOPE,
|
|
15
|
-
is_ast_same_or_ancestor_of,
|
|
16
|
-
)
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
@dataclass
|
|
20
|
-
class MemoryRecord:
|
|
21
|
-
id: int
|
|
22
|
-
name: str
|
|
23
|
-
key: KEY
|
|
24
|
-
scope: SCOPE
|
|
25
|
-
type: REGISTER_TYPE
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
@dataclass
|
|
29
|
-
class Memory:
|
|
30
|
-
counter: int = 0
|
|
31
|
-
registers: pd.DataFrame = field(
|
|
32
|
-
default_factory=lambda: pd.DataFrame(
|
|
33
|
-
columns=["id", "name", "key", "scope", "type"],
|
|
34
|
-
)
|
|
35
|
-
)
|
|
36
|
-
|
|
37
|
-
def register(
|
|
38
|
-
self,
|
|
39
|
-
name: str,
|
|
40
|
-
key: KEY,
|
|
41
|
-
scope: SCOPE,
|
|
42
|
-
type: REGISTER_TYPE,
|
|
43
|
-
strict: bool,
|
|
44
|
-
):
|
|
45
|
-
"""
|
|
46
|
-
Args:
|
|
47
|
-
strict: if True, raise an error if there is a register with the same name, key, and type.
|
|
48
|
-
"""
|
|
49
|
-
if strict:
|
|
50
|
-
matched_records = self.find(name=name, key=key, type=type)
|
|
51
|
-
else:
|
|
52
|
-
matched_records = self.find(name=name, key=key, scope=scope, type=type)
|
|
53
|
-
|
|
54
|
-
if len(matched_records) > 0:
|
|
55
|
-
raise ValueError(
|
|
56
|
-
f"There is already a register with the same name={name}, key={key}, scope={scope} (if strict=True), and type={type}"
|
|
57
|
-
)
|
|
58
|
-
|
|
59
|
-
register_id = self.counter
|
|
60
|
-
self.counter += 1
|
|
61
|
-
self.registers.loc[register_id] = [register_id, name, key, scope, type]
|
|
62
|
-
return register_id
|
|
63
|
-
|
|
64
|
-
def remove_one(self, register_id: int) -> None:
|
|
65
|
-
self.registers = self.registers.drop(register_id)
|
|
66
|
-
|
|
67
|
-
def get_one(self, register_id: int) -> MemoryRecord:
|
|
68
|
-
reg = self.registers.loc[register_id]
|
|
69
|
-
return MemoryRecord(
|
|
70
|
-
reg["id"], reg["name"], reg["key"], reg["scope"], reg["type"]
|
|
71
|
-
)
|
|
72
|
-
|
|
73
|
-
def update_scope(self, register_id: int, scope: SCOPE):
|
|
74
|
-
self.registers.at[register_id, "scope"] = scope
|
|
75
|
-
|
|
76
|
-
def find(
|
|
77
|
-
self,
|
|
78
|
-
*,
|
|
79
|
-
name: Optional[str] = None,
|
|
80
|
-
key: Optional[KEY] = None,
|
|
81
|
-
scope: Optional[SCOPE] = None,
|
|
82
|
-
type: Optional[REGISTER_TYPE] = None,
|
|
83
|
-
) -> list[MemoryRecord]:
|
|
84
|
-
conditions = []
|
|
85
|
-
if name is not None:
|
|
86
|
-
conditions.append(self.registers["name"] == name)
|
|
87
|
-
if key is not None:
|
|
88
|
-
conditions.append(self.registers["key"] == key)
|
|
89
|
-
if scope is not None:
|
|
90
|
-
conditions.append(self.registers["scope"] == scope)
|
|
91
|
-
if type is not None:
|
|
92
|
-
conditions.append(self.registers["type"] == type)
|
|
93
|
-
|
|
94
|
-
assert len(conditions) > 0
|
|
95
|
-
query = conditions[0]
|
|
96
|
-
for cond in conditions[1:]:
|
|
97
|
-
query = query & cond
|
|
98
|
-
|
|
99
|
-
res = self.registers[query]
|
|
100
|
-
out = []
|
|
101
|
-
for i in range(len(res)):
|
|
102
|
-
x = res.iloc[i]
|
|
103
|
-
out.append(
|
|
104
|
-
MemoryRecord(x["id"], x["name"], x["key"], x["scope"], x["type"])
|
|
105
|
-
)
|
|
106
|
-
return out
|
|
107
|
-
|
|
108
|
-
def find_one(
|
|
109
|
-
self,
|
|
110
|
-
*,
|
|
111
|
-
name: Optional[str] = None,
|
|
112
|
-
key: Optional[KEY] = None,
|
|
113
|
-
scope: Optional[SCOPE] = None,
|
|
114
|
-
type: Optional[REGISTER_TYPE] = None,
|
|
115
|
-
) -> MemoryRecord:
|
|
116
|
-
res = self.find(name=name, key=key, scope=scope, type=type)
|
|
117
|
-
if len(res) == 0:
|
|
118
|
-
raise ValueError(
|
|
119
|
-
f"Cannot find a register matched name={name}, key={key}, scope={scope}, type={type}"
|
|
120
|
-
)
|
|
121
|
-
if len(res) > 1:
|
|
122
|
-
raise ValueError(
|
|
123
|
-
f"Found multiple registers matched name={name}, key={key}, scope={scope}, type={type}"
|
|
124
|
-
)
|
|
125
|
-
return res[0]
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
class VarScope(NamedTuple):
|
|
129
|
-
ast: AST_ID # the ast that has a child, which is an assign statement, that the first time the variable is defined
|
|
130
|
-
child_index: int # the index of the child (assign statement)
|
|
131
|
-
|
|
132
|
-
def is_subscope_of(self, scope: VarScope) -> bool:
|
|
133
|
-
if scope.ast == self.ast:
|
|
134
|
-
return scope.child_index <= self.child_index
|
|
135
|
-
return is_ast_same_or_ancestor_of(scope.ast, self.ast)
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
@dataclass
|
|
139
|
-
class Var: # variable
|
|
140
|
-
name: str
|
|
141
|
-
key: Optional[KEY]
|
|
142
|
-
register_id: int
|
|
143
|
-
scope: Optional[VarScope]
|
|
144
|
-
|
|
145
|
-
def get_name(self) -> str:
|
|
146
|
-
return f"{self.name}_{self.register_id}"
|
|
147
|
-
|
|
148
|
-
def set_scope(self, mem: Memory, scope: VarScope):
|
|
149
|
-
assert self.scope == None, "Cannot change the scope of a variable"
|
|
150
|
-
assert (
|
|
151
|
-
mem.get_one(self.register_id).scope == NO_SCOPE
|
|
152
|
-
), "Cannot change the scope of a variable"
|
|
153
|
-
|
|
154
|
-
self.scope = scope
|
|
155
|
-
mem.update_scope(self.register_id, scope)
|
|
156
|
-
|
|
157
|
-
def delete(self, mem: Memory):
|
|
158
|
-
mem.remove_one(self.register_id)
|
|
159
|
-
|
|
160
|
-
@staticmethod
|
|
161
|
-
def create(
|
|
162
|
-
mem: Memory,
|
|
163
|
-
name: str,
|
|
164
|
-
key: KEY = NO_KEY,
|
|
165
|
-
scope: Optional[VarScope] = None,
|
|
166
|
-
strict: bool = True,
|
|
167
|
-
):
|
|
168
|
-
return Var(
|
|
169
|
-
name,
|
|
170
|
-
key,
|
|
171
|
-
mem.register(name, key, scope or NO_SCOPE, "var", strict),
|
|
172
|
-
scope,
|
|
173
|
-
)
|
|
174
|
-
|
|
175
|
-
@staticmethod
|
|
176
|
-
def deref(
|
|
177
|
-
mem: Memory,
|
|
178
|
-
*,
|
|
179
|
-
name: Optional[str] = None,
|
|
180
|
-
key: Optional[KEY] = None,
|
|
181
|
-
scope: Optional[VarScope] = None,
|
|
182
|
-
) -> Var:
|
|
183
|
-
reg = mem.find_one(name=name, key=key, scope=scope, type="var")
|
|
184
|
-
if reg.scope == NO_SCOPE:
|
|
185
|
-
scope = None
|
|
186
|
-
else:
|
|
187
|
-
scope = cast(VarScope, reg.scope)
|
|
188
|
-
return Var(reg.name, reg.key, reg.id, scope)
|
|
189
|
-
|
|
190
|
-
@staticmethod
|
|
191
|
-
def find_by_scope(mem: Memory, scope: VarScope) -> list[Var]:
|
|
192
|
-
"""Find all variables that were defined in the given scope."""
|
|
193
|
-
vars = []
|
|
194
|
-
for reg in mem.find(type="var"):
|
|
195
|
-
if reg.scope != NO_SCOPE and cast(VarScope, reg.scope).is_subscope_of(
|
|
196
|
-
scope
|
|
197
|
-
):
|
|
198
|
-
vars.append(Var(reg.name, reg.key, reg.id, cast(VarScope, reg.scope)))
|
|
199
|
-
return vars
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from dataclasses import dataclass, field
|
|
4
|
-
|
|
5
|
-
from codegen.models.ast import AST
|
|
6
|
-
from codegen.models.memory import Memory
|
|
7
|
-
from codegen.models.statement import BlockStatement, ImportStatement
|
|
8
|
-
from codegen.models.types import AST_ID
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
@dataclass(init=False)
|
|
12
|
-
class Program:
|
|
13
|
-
root: AST
|
|
14
|
-
memory: Memory
|
|
15
|
-
import_manager: ImportManager
|
|
16
|
-
|
|
17
|
-
def __init__(self):
|
|
18
|
-
self.import_manager = ImportManager()
|
|
19
|
-
self.memory = Memory()
|
|
20
|
-
|
|
21
|
-
self.root = AST.root()
|
|
22
|
-
self.root._add_stmt(self.import_manager)
|
|
23
|
-
|
|
24
|
-
def get_ast_for_id(self, id: AST_ID) -> AST:
|
|
25
|
-
ast = self.root.find_ast(id)
|
|
26
|
-
if ast is None:
|
|
27
|
-
raise KeyError(id)
|
|
28
|
-
return ast
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
@dataclass
|
|
32
|
-
class ImportManager(BlockStatement):
|
|
33
|
-
|
|
34
|
-
stmts: list[ImportStatement] = field(default_factory=list)
|
|
35
|
-
|
|
36
|
-
def import_(self, module: str):
|
|
37
|
-
if not any(stmt.module == module for stmt in self.stmts):
|
|
38
|
-
self.stmts.append(ImportStatement(module))
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from typing import Literal, TypeAlias
|
|
4
|
-
|
|
5
|
-
KEY = str | tuple
|
|
6
|
-
NO_KEY = tuple()
|
|
7
|
-
REGISTER_TYPE = Literal["var"]
|
|
8
|
-
|
|
9
|
-
AST_ID: TypeAlias = str
|
|
10
|
-
NO_AST_ID: AST_ID = ""
|
|
11
|
-
|
|
12
|
-
SCOPE: TypeAlias = tuple
|
|
13
|
-
NO_SCOPE: SCOPE = tuple()
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def is_ast_same_or_ancestor_of(ancestor: AST_ID, descendant: AST_ID):
|
|
17
|
-
"""Check whether ast is an ancestor of self"""
|
|
18
|
-
return ancestor == descendant or descendant.startswith(ancestor)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|