codegen-2 2.16.5__tar.gz → 2.16.7__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: codegen-2
3
- Version: 2.16.5
3
+ Version: 2.16.7
4
4
  Summary: Meta-programming
5
5
  Author: Binh Vu
6
6
  Author-email: Binh Vu <binh@toan2.com>
@@ -28,6 +28,7 @@ from codegen.models.statement import (
28
28
  from codegen.models.types import AstID
29
29
  from codegen.models.utils import move_ast_id
30
30
  from codegen.models.var import DeferredVar, Var, VarScope
31
+ from codegen.parsers.python import add_python_codeblock
31
32
 
32
33
  if TYPE_CHECKING:
33
34
  from codegen.models.program import Program
@@ -195,9 +196,9 @@ class AST:
195
196
 
196
197
  def for_loop(self, item: DeferredVar, iter: Expr):
197
198
  """When we construct a for-loop, the item (or var) is always declared. We don't allow reassigning the variable in the for-loop."""
198
- assert isinstance(item, DeferredVar) and item.has_not_been_created(), (
199
- "The item must be a new variable"
200
- )
199
+ assert (
200
+ isinstance(item, DeferredVar) and item.has_not_been_created()
201
+ ), "The item must be a new variable"
201
202
  var = self.prog.create_var(
202
203
  item.name, item.key, self.next_grandchild_id(), item.force_name
203
204
  )
@@ -228,6 +229,12 @@ class AST:
228
229
  def python_stmt(self, stmt: str) -> AST:
229
230
  return self._add_stmt(PythonStatement(stmt))
230
231
 
232
+ def parse_python_codeblock(
233
+ self, codeblock: str, *, global_imports: bool = True
234
+ ) -> AST:
235
+ add_python_codeblock(self, codeblock, global_imports=global_imports)
236
+ return self
237
+
231
238
  def update_recursively(
232
239
  self, fn: Callable[[AST, Any], tuple[AST, Any, bool]], context: Any
233
240
  ):
@@ -30,6 +30,13 @@ class StandardExceptionExpr(ExceptionExpr):
30
30
  cls: Expr
31
31
  args: Sequence[Expr]
32
32
 
33
+ @staticmethod
34
+ def simple(clsname: str, msg: str | Expr):
35
+ return StandardExceptionExpr(
36
+ ExprIdent(clsname),
37
+ [ExprConstant(msg) if isinstance(msg, str) else msg],
38
+ )
39
+
33
40
  def to_python(self):
34
41
  return f"{self.cls.to_python()}({', '.join([arg.to_python() for arg in self.args])})"
35
42
 
@@ -0,0 +1,15 @@
1
+ from __future__ import annotations
2
+
3
+ from typing import Callable, Literal, Optional, Protocol
4
+
5
+ KEY = str | tuple
6
+ NO_KEY = tuple()
7
+ REGISTER_TYPE = Literal["var"]
8
+
9
+ AstID = tuple[int, ...]
10
+
11
+
12
+ class ImportFn(Protocol):
13
+ def __call__(
14
+ self, module: str, is_import_attr: bool, alias: Optional[str] = None
15
+ ) -> None: ...
@@ -0,0 +1,3 @@
1
+ from codegen.parsers.python import add_python_codeblock
2
+
3
+ __all__ = ["add_python_codeblock"]
@@ -0,0 +1,440 @@
1
+ from __future__ import annotations
2
+
3
+ import ast as py_ast
4
+ import textwrap
5
+ from typing import TYPE_CHECKING, Optional, Sequence
6
+
7
+ from codegen.models.expr import (
8
+ Expr,
9
+ ExprAwait,
10
+ ExprConstant,
11
+ ExprDivision,
12
+ ExprEqual,
13
+ ExprFuncCall,
14
+ ExprIdent,
15
+ ExprIs,
16
+ ExprLessThanOrEqual,
17
+ ExprLogicalAnd,
18
+ ExprLogicalOr,
19
+ ExprMethodCall,
20
+ ExprNegation,
21
+ ExprNotEqual,
22
+ ExprTernary,
23
+ ExprVar,
24
+ PredefinedFn,
25
+ )
26
+ from codegen.models.statement import (
27
+ AssertionStatement,
28
+ AssignStatement,
29
+ BreakStatement,
30
+ ContinueStatement,
31
+ ExceptionStatement,
32
+ NoStatement,
33
+ PythonDecoratorStatement,
34
+ )
35
+ from codegen.models.var import DeferredVar, Var
36
+
37
+ if TYPE_CHECKING:
38
+ from codegen.models.ast import AST
39
+
40
+
41
+ def add_python_codeblock(target: AST, codeblock: str, *, global_imports: bool = True):
42
+ parser = _PythonCodeblockParser(global_imports=global_imports)
43
+ tree = py_ast.parse(_normalize_codeblock(codeblock))
44
+ parser.parse_body(target, tree.body)
45
+
46
+
47
+ def _normalize_codeblock(codeblock: str) -> str:
48
+ code = textwrap.dedent(codeblock).strip()
49
+ lines = code.splitlines()
50
+
51
+ if lines and lines[0].strip().startswith("```"):
52
+ if not lines[-1].strip().startswith("```"):
53
+ raise ValueError("Python codeblock fence is not closed")
54
+ code = "\n".join(lines[1:-1])
55
+
56
+ return textwrap.dedent(code).strip("\n")
57
+
58
+
59
+ class _PythonCodeblockParser:
60
+ def __init__(self, *, global_imports: bool):
61
+ self.global_imports = global_imports
62
+
63
+ def parse_body(self, target: AST, body: Sequence[py_ast.stmt]) -> None:
64
+ for node in body:
65
+ self.parse_stmt(target, node)
66
+
67
+ def parse_stmt(self, target: AST, node: py_ast.stmt) -> None:
68
+ if isinstance(node, py_ast.Import):
69
+ self._parse_import(target, node)
70
+ elif isinstance(node, py_ast.ImportFrom):
71
+ self._parse_import_from(target, node)
72
+ elif isinstance(node, py_ast.Assign):
73
+ self._parse_assign(target, node)
74
+ elif isinstance(node, py_ast.Return):
75
+ self._parse_return(target, node)
76
+ elif isinstance(node, py_ast.Expr):
77
+ target.expr(self.parse_expr(target, node.value))
78
+ elif isinstance(node, py_ast.If):
79
+ self._parse_if(target, node)
80
+ elif isinstance(node, py_ast.For):
81
+ self._parse_for(target, node)
82
+ elif isinstance(node, py_ast.FunctionDef):
83
+ self._parse_function(target, node, is_async=False)
84
+ elif isinstance(node, py_ast.AsyncFunctionDef):
85
+ self._parse_function(target, node, is_async=True)
86
+ elif isinstance(node, py_ast.ClassDef):
87
+ self._parse_class(target, node)
88
+ elif isinstance(node, py_ast.Try):
89
+ self._parse_try(target, node)
90
+ elif isinstance(node, py_ast.Assert):
91
+ self._parse_assert(target, node)
92
+ elif isinstance(node, py_ast.Pass):
93
+ target._add_stmt(NoStatement())
94
+ elif isinstance(node, py_ast.Break):
95
+ target._add_stmt(BreakStatement())
96
+ elif isinstance(node, py_ast.Continue):
97
+ target._add_stmt(ContinueStatement())
98
+ elif isinstance(node, py_ast.Raise):
99
+ self._parse_raise(target, node)
100
+ else:
101
+ raise self._unsupported(node)
102
+
103
+ def parse_expr(self, target: AST, node: py_ast.expr) -> Expr:
104
+ if isinstance(node, py_ast.Constant):
105
+ return self._parse_constant(node)
106
+ if isinstance(node, py_ast.Name):
107
+ var = self._find_var(target, node.id)
108
+ if var is not None:
109
+ return ExprVar(var)
110
+ return ExprIdent(node.id)
111
+ if isinstance(node, py_ast.Call):
112
+ return self._parse_call(target, node)
113
+ if isinstance(node, py_ast.Attribute):
114
+ return PredefinedFn.attr_getter(
115
+ self.parse_expr(target, node.value), ExprIdent(node.attr)
116
+ )
117
+ if isinstance(node, py_ast.Subscript):
118
+ return PredefinedFn.item_getter(
119
+ self.parse_expr(target, node.value),
120
+ self._parse_slice(target, node.slice),
121
+ )
122
+ if isinstance(node, py_ast.List):
123
+ return PredefinedFn.list(
124
+ [self.parse_expr(target, item) for item in node.elts]
125
+ )
126
+ if isinstance(node, py_ast.Tuple):
127
+ return PredefinedFn.tuple(
128
+ [self.parse_expr(target, item) for item in node.elts]
129
+ )
130
+ if isinstance(node, py_ast.Set):
131
+ return PredefinedFn.set(
132
+ [self.parse_expr(target, item) for item in node.elts]
133
+ )
134
+ if isinstance(node, py_ast.Dict):
135
+ return self._parse_dict(target, node)
136
+ if isinstance(node, py_ast.Await):
137
+ return ExprAwait(self.parse_expr(target, node.value))
138
+ if isinstance(node, py_ast.BoolOp):
139
+ return self._parse_bool_op(target, node)
140
+ if isinstance(node, py_ast.UnaryOp):
141
+ return self._parse_unary_op(target, node)
142
+ if isinstance(node, py_ast.IfExp):
143
+ return ExprTernary(
144
+ self.parse_expr(target, node.test),
145
+ self.parse_expr(target, node.body),
146
+ self.parse_expr(target, node.orelse),
147
+ )
148
+ if isinstance(node, py_ast.Compare):
149
+ return self._parse_compare(target, node)
150
+ if isinstance(node, py_ast.BinOp):
151
+ return self._parse_bin_op(target, node)
152
+ raise self._unsupported(node)
153
+
154
+ def _parse_import(self, target: AST, node: py_ast.Import) -> None:
155
+ for alias in node.names:
156
+ self._add_import(target, alias.name, False, alias.asname)
157
+
158
+ def _parse_import_from(self, target: AST, node: py_ast.ImportFrom) -> None:
159
+ if node.level != 0 or node.module is None:
160
+ raise self._unsupported(node, "relative imports are not supported")
161
+
162
+ for alias in node.names:
163
+ self._add_import(target, f"{node.module}.{alias.name}", True, alias.asname)
164
+
165
+ def _add_import(
166
+ self, target: AST, module: str, is_import_attr: bool, alias: Optional[str]
167
+ ) -> None:
168
+ if self.global_imports:
169
+ target.prog.import_(module, is_import_attr, alias)
170
+ else:
171
+ target.import_(module, is_import_attr, alias)
172
+
173
+ def _parse_assign(self, target: AST, node: py_ast.Assign) -> None:
174
+ if len(node.targets) != 1:
175
+ raise self._unsupported(
176
+ node, "multiple assignment targets are not supported"
177
+ )
178
+
179
+ expr = self.parse_expr(target, node.value)
180
+ assign_target = node.targets[0]
181
+ if isinstance(assign_target, py_ast.Name):
182
+ var = self._find_var(target, assign_target.id)
183
+ if var is None:
184
+ target.assign(DeferredVar.simple(assign_target.id), expr)
185
+ else:
186
+ target.assign(var, expr)
187
+ return
188
+
189
+ target_expr = self._parse_assign_target(target, assign_target)
190
+ target._add_stmt(AssignStatement(target_expr, expr))
191
+
192
+ def _parse_assign_target(self, target: AST, node: py_ast.expr) -> Expr:
193
+ if isinstance(node, py_ast.Attribute):
194
+ return PredefinedFn.attr_getter(
195
+ self.parse_expr(target, node.value), ExprIdent(node.attr)
196
+ )
197
+ if isinstance(node, py_ast.Subscript):
198
+ return PredefinedFn.item_getter(
199
+ self.parse_expr(target, node.value),
200
+ self._parse_slice(target, node.slice),
201
+ )
202
+ raise self._unsupported(node, "assignment target is not supported")
203
+
204
+ def _parse_return(self, target: AST, node: py_ast.Return) -> None:
205
+ if node.value is None:
206
+ raise self._unsupported(node, "bare return is not supported")
207
+ target.return_(self.parse_expr(target, node.value))
208
+
209
+ def _parse_if(self, target: AST, node: py_ast.If) -> None:
210
+ if_ast = target.if_(self.parse_expr(target, node.test))
211
+ self.parse_body(if_ast, node.body)
212
+ if node.orelse:
213
+ else_ast = target.else_()
214
+ self.parse_body(else_ast, node.orelse)
215
+
216
+ def _parse_for(self, target: AST, node: py_ast.For) -> None:
217
+ if node.orelse:
218
+ raise self._unsupported(node, "for/else is not supported")
219
+ if not isinstance(node.target, py_ast.Name):
220
+ raise self._unsupported(node.target, "for target is not supported")
221
+
222
+ for_ast = target.for_loop(
223
+ DeferredVar.simple(node.target.id), self.parse_expr(target, node.iter)
224
+ )
225
+ self.parse_body(for_ast, node.body)
226
+
227
+ def _parse_function(
228
+ self,
229
+ target: AST,
230
+ node: py_ast.FunctionDef | py_ast.AsyncFunctionDef,
231
+ *,
232
+ is_async: bool,
233
+ ) -> None:
234
+ self._add_decorators(target, node.decorator_list)
235
+ return_type = (
236
+ self.parse_expr(target, node.returns) if node.returns is not None else None
237
+ )
238
+ fn_ast = target.func(
239
+ node.name,
240
+ self._parse_arguments(target, node.args),
241
+ return_type=return_type,
242
+ is_async=is_async,
243
+ )
244
+ self.parse_body(fn_ast, node.body)
245
+
246
+ def _parse_class(self, target: AST, node: py_ast.ClassDef) -> None:
247
+ if node.keywords:
248
+ raise self._unsupported(node, "class keywords are not supported")
249
+
250
+ self._add_decorators(target, node.decorator_list)
251
+ class_ast = target.class_(
252
+ node.name, [self.parse_expr(target, base) for base in node.bases]
253
+ )
254
+ self.parse_body(class_ast, node.body)
255
+
256
+ def _parse_try(self, target: AST, node: py_ast.Try) -> None:
257
+ if node.orelse:
258
+ raise self._unsupported(node, "try/else is not supported")
259
+ if node.finalbody:
260
+ raise self._unsupported(node, "try/finally is not supported")
261
+
262
+ try_ast = target.try_()
263
+ self.parse_body(try_ast, node.body)
264
+ for handler in node.handlers:
265
+ if handler.name is not None:
266
+ raise self._unsupported(handler, "except aliases are not supported")
267
+ catch_ast = target.catch(
268
+ self.parse_expr(target, handler.type)
269
+ if handler.type is not None
270
+ else None
271
+ )
272
+ self.parse_body(catch_ast, handler.body)
273
+
274
+ def _parse_assert(self, target: AST, node: py_ast.Assert) -> None:
275
+ target._add_stmt(
276
+ AssertionStatement(
277
+ self.parse_expr(target, node.test),
278
+ self.parse_expr(target, node.msg) if node.msg is not None else None,
279
+ )
280
+ )
281
+
282
+ def _parse_raise(self, target: AST, node: py_ast.Raise) -> None:
283
+ if node.exc is None:
284
+ raise self._unsupported(node, "bare raise is not supported")
285
+ if node.cause is not None:
286
+ raise self._unsupported(node, "raise from is not supported")
287
+ target._add_stmt(ExceptionStatement(self.parse_expr(target, node.exc)))
288
+
289
+ def _parse_arguments(
290
+ self, target: AST, arguments: py_ast.arguments
291
+ ) -> list[DeferredVar | tuple[DeferredVar, Expr]]:
292
+ if arguments.posonlyargs:
293
+ raise self._unsupported(
294
+ arguments, "positional-only arguments are not supported"
295
+ )
296
+ if arguments.vararg is not None:
297
+ raise self._unsupported(arguments.vararg, "*args is not supported")
298
+ if arguments.kwonlyargs:
299
+ raise self._unsupported(
300
+ arguments, "keyword-only arguments are not supported"
301
+ )
302
+ if arguments.kwarg is not None:
303
+ raise self._unsupported(arguments.kwarg, "**kwargs is not supported")
304
+
305
+ default_offset = len(arguments.args) - len(arguments.defaults)
306
+ parsed_args: list[DeferredVar | tuple[DeferredVar, Expr]] = []
307
+ for index, arg in enumerate(arguments.args):
308
+ if arg.type_comment is not None:
309
+ raise self._unsupported(arg, "argument type comments are not supported")
310
+
311
+ var = DeferredVar.simple(
312
+ arg.arg,
313
+ (
314
+ self.parse_expr(target, arg.annotation)
315
+ if arg.annotation is not None
316
+ else None
317
+ ),
318
+ )
319
+ default_index = index - default_offset
320
+ if default_index >= 0:
321
+ parsed_args.append(
322
+ (var, self.parse_expr(target, arguments.defaults[default_index]))
323
+ )
324
+ else:
325
+ parsed_args.append(var)
326
+ return parsed_args
327
+
328
+ def _add_decorators(self, target: AST, decorators: Sequence[py_ast.expr]) -> None:
329
+ for decorator in decorators:
330
+ expr = self.parse_expr(target, decorator)
331
+ if isinstance(expr, ExprFuncCall):
332
+ target._add_stmt(
333
+ PythonDecoratorStatement(
334
+ expr, render_call_parentheses=isinstance(decorator, py_ast.Call)
335
+ )
336
+ )
337
+ else:
338
+ target._add_stmt(PythonDecoratorStatement(ExprFuncCall(expr, [])))
339
+
340
+ def _parse_call(self, target: AST, node: py_ast.Call) -> Expr:
341
+ args = [self.parse_expr(target, arg) for arg in node.args]
342
+ for keyword in node.keywords:
343
+ if keyword.arg is None:
344
+ raise self._unsupported(keyword, "** call arguments are not supported")
345
+ args.append(
346
+ PredefinedFn.keyword_assignment(
347
+ keyword.arg, self.parse_expr(target, keyword.value)
348
+ )
349
+ )
350
+
351
+ if isinstance(node.func, py_ast.Attribute):
352
+ return ExprMethodCall(
353
+ self.parse_expr(target, node.func.value), node.func.attr, args
354
+ )
355
+ return ExprFuncCall(self.parse_expr(target, node.func), args)
356
+
357
+ def _parse_constant(self, node: py_ast.Constant) -> ExprConstant:
358
+ if isinstance(node.value, (bool, str, int, float)) or node.value is None:
359
+ return ExprConstant(node.value)
360
+ raise self._unsupported(node, "constant type is not supported")
361
+
362
+ def _parse_slice(self, target: AST, node: py_ast.expr | py_ast.slice) -> Expr:
363
+ if isinstance(node, py_ast.Slice):
364
+ raise self._unsupported(node, "slice ranges are not supported")
365
+ return self.parse_expr(target, node)
366
+
367
+ def _parse_dict(self, target: AST, node: py_ast.Dict) -> Expr:
368
+ items = []
369
+ for key, value in zip(node.keys, node.values):
370
+ if key is None:
371
+ raise self._unsupported(node, "** dict entries are not supported")
372
+ items.append((self.parse_expr(target, key), self.parse_expr(target, value)))
373
+ return PredefinedFn.dict(items)
374
+
375
+ def _parse_bool_op(self, target: AST, node: py_ast.BoolOp) -> Expr:
376
+ terms = [self.parse_expr(target, value) for value in node.values]
377
+ if isinstance(node.op, py_ast.And):
378
+ return ExprLogicalAnd(terms)
379
+ if isinstance(node.op, py_ast.Or):
380
+ return ExprLogicalOr(terms)
381
+ raise self._unsupported(node)
382
+
383
+ def _parse_unary_op(self, target: AST, node: py_ast.UnaryOp) -> Expr:
384
+ if isinstance(node.op, py_ast.Not):
385
+ return ExprNegation(self.parse_expr(target, node.operand))
386
+ raise self._unsupported(node, "unary operator is not supported")
387
+
388
+ def _parse_compare(self, target: AST, node: py_ast.Compare) -> Expr:
389
+ if len(node.ops) != 1 or len(node.comparators) != 1:
390
+ raise self._unsupported(node, "chained comparisons are not supported")
391
+
392
+ left = self.parse_expr(target, node.left)
393
+ right = self.parse_expr(target, node.comparators[0])
394
+ op = node.ops[0]
395
+ if isinstance(op, py_ast.Eq):
396
+ return ExprEqual(left, right)
397
+ if isinstance(op, py_ast.NotEq):
398
+ return ExprNotEqual(left, right)
399
+ if isinstance(op, py_ast.Lt):
400
+ return ExprLessThanOrEqual(left, right)
401
+ if isinstance(op, py_ast.Is):
402
+ return ExprIs(left, right)
403
+ if isinstance(op, py_ast.IsNot):
404
+ return ExprNegation(ExprIs(left, right))
405
+ if isinstance(op, py_ast.In):
406
+ return PredefinedFn.has_item(right, left)
407
+ if isinstance(op, py_ast.NotIn):
408
+ return PredefinedFn.not_has_item(right, left)
409
+ raise self._unsupported(node, "comparison operator is not supported")
410
+
411
+ def _parse_bin_op(self, target: AST, node: py_ast.BinOp) -> Expr:
412
+ if isinstance(node.op, py_ast.Div):
413
+ return ExprDivision(
414
+ self.parse_expr(target, node.left), self.parse_expr(target, node.right)
415
+ )
416
+ raise self._unsupported(node, "binary operator is not supported")
417
+
418
+ def _find_var(self, target: AST, name: str) -> Optional[Var]:
419
+ reg = target.prog.vars.find((name,), target.next_child_id())
420
+ if reg is None:
421
+ return None
422
+ return Var(
423
+ name=reg.name,
424
+ key=reg.key,
425
+ register_id=reg.id,
426
+ scope=reg.scope,
427
+ force_name=reg.force_name,
428
+ type=reg.type,
429
+ )
430
+
431
+ def _unsupported(
432
+ self, node: py_ast.AST, reason: Optional[str] = None
433
+ ) -> NotImplementedError:
434
+ location = ""
435
+ if hasattr(node, "lineno"):
436
+ location = f" at line {node.lineno}, column {node.col_offset}"
437
+ detail = f": {reason}" if reason is not None else ""
438
+ return NotImplementedError(
439
+ f"Unsupported Python AST {node.__class__.__name__}{location}{detail}"
440
+ )
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "codegen-2"
3
- version = "2.16.5"
3
+ version = "2.16.7"
4
4
  description = "Meta-programming"
5
5
  authors = [{ name = "Binh Vu", email = "binh@toan2.com" }]
6
6
  license = { text = "MIT" }
@@ -1,11 +0,0 @@
1
- from __future__ import annotations
2
-
3
- from typing import Callable, Literal
4
-
5
- KEY = str | tuple
6
- NO_KEY = tuple()
7
- REGISTER_TYPE = Literal["var"]
8
-
9
- AstID = tuple[int, ...]
10
-
11
- ImportFn = Callable[[str, bool, str], None] | Callable[[str, bool], None]
File without changes