code-towel 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- code_towel-1.0.0.dist-info/METADATA +722 -0
- code_towel-1.0.0.dist-info/RECORD +27 -0
- code_towel-1.0.0.dist-info/WHEEL +5 -0
- code_towel-1.0.0.dist-info/entry_points.txt +4 -0
- code_towel-1.0.0.dist-info/licenses/LICENSE +201 -0
- code_towel-1.0.0.dist-info/top_level.txt +1 -0
- towel/__init__.py +20 -0
- towel/cli.py +813 -0
- towel/unification/__init__.py +24 -0
- towel/unification/assignment_analyzer.py +403 -0
- towel/unification/ast_normalizer.py +223 -0
- towel/unification/ast_pretty_printer.py +133 -0
- towel/unification/binding_detector.py +381 -0
- towel/unification/block_signature.py +123 -0
- towel/unification/builtins.py +209 -0
- towel/unification/exceptions.py +113 -0
- towel/unification/extractor.py +884 -0
- towel/unification/models.py +190 -0
- towel/unification/nominal_unifier.py +375 -0
- towel/unification/orphan_detector.py +187 -0
- towel/unification/pipeline.py +524 -0
- towel/unification/project_layout.py +201 -0
- towel/unification/refactor_engine.py +3381 -0
- towel/unification/scope_analyzer.py +597 -0
- towel/unification/unifier.py +2128 -0
- towel/unification/visitor_utils.py +83 -0
- towel/unification/visitors.py +377 -0
|
@@ -0,0 +1,2128 @@
|
|
|
1
|
+
# Copyright 2025 Eric Allen
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""
|
|
16
|
+
Unification algorithm for finding parameterizable differences in AST nodes.
|
|
17
|
+
|
|
18
|
+
This is based on Robinson's unification algorithm from automated theorem proving,
|
|
19
|
+
adapted for AST comparison.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
import ast
|
|
23
|
+
from typing import Callable, Dict, Optional, List, Tuple, Set, Any, cast, Sequence, Union, Iterator
|
|
24
|
+
from dataclasses import dataclass, field
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass
|
|
28
|
+
class Substitution:
|
|
29
|
+
"""
|
|
30
|
+
Represents a substitution mapping from sub-expressions to parameter names.
|
|
31
|
+
|
|
32
|
+
Each entry maps a (block_index, sub_expression) to a parameter name.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
mappings: Dict[Tuple[int, str], str] = field(default_factory=dict)
|
|
36
|
+
# Maps parameter names to the list of expressions they replace
|
|
37
|
+
param_expressions: Dict[str, List[Tuple[int, ast.AST]]] = field(default_factory=dict)
|
|
38
|
+
# Maps parameter names to list of bound variables they should take as function args
|
|
39
|
+
# If a parameter is in this dict, it should be a function parameter
|
|
40
|
+
function_params: Dict[str, List[str]] = field(default_factory=dict)
|
|
41
|
+
# Optional hygienic renames captured during unification (one mapping per block)
|
|
42
|
+
hygienic_renames: Optional[List[Dict[str, str]]] = field(default_factory=list)
|
|
43
|
+
# Parameters that, after substitution, are used in call position (as a callee)
|
|
44
|
+
# within the extracted function body. These should be passed as thunks (lambdas)
|
|
45
|
+
# that perform the call to avoid eager evaluation at the call site.
|
|
46
|
+
params_used_as_callee: Set[str] = field(default_factory=set)
|
|
47
|
+
|
|
48
|
+
# Optional: parameters introduced post-unification to promote literal arguments
|
|
49
|
+
# of higher-order factory calls (e.g., make_validator(5)) into threaded parameters
|
|
50
|
+
# of the extracted helper, even when those literals do not differ across blocks.
|
|
51
|
+
#
|
|
52
|
+
# Schema:
|
|
53
|
+
# promoted_literal_args[param_name][block_idx] = ast.AST (expression to pass)
|
|
54
|
+
#
|
|
55
|
+
# This allows call generation to supply the original literal (or expression)
|
|
56
|
+
# per block for such promoted parameters. By default, this dict is empty and
|
|
57
|
+
# has no effect on behavior until a promotion pass populates it.
|
|
58
|
+
promoted_literal_args: Dict[str, Dict[int, ast.AST]] = field(default_factory=dict)
|
|
59
|
+
|
|
60
|
+
def add_mapping(
|
|
61
|
+
self, block_idx: int, expr: ast.AST, param_name: str, bound_vars: Optional[List[str]] = None
|
|
62
|
+
) -> None:
|
|
63
|
+
"""
|
|
64
|
+
Add a mapping from an expression to a parameter name.
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
block_idx: Block index
|
|
68
|
+
expr: Expression being parameterized
|
|
69
|
+
param_name: Name of the parameter
|
|
70
|
+
bound_vars: List of bound variables the expression references (for function parameters)
|
|
71
|
+
"""
|
|
72
|
+
expr_str = ast.unparse(expr)
|
|
73
|
+
key = (block_idx, expr_str)
|
|
74
|
+
self.mappings[key] = param_name
|
|
75
|
+
|
|
76
|
+
if param_name not in self.param_expressions:
|
|
77
|
+
self.param_expressions[param_name] = []
|
|
78
|
+
self.param_expressions[param_name].append((block_idx, expr))
|
|
79
|
+
|
|
80
|
+
# If this expression references bound variables, mark parameter as function
|
|
81
|
+
if bound_vars:
|
|
82
|
+
if param_name not in self.function_params:
|
|
83
|
+
self.function_params[param_name] = bound_vars
|
|
84
|
+
else:
|
|
85
|
+
# Merge bound variables (should be same across all blocks)
|
|
86
|
+
existing = set(self.function_params[param_name])
|
|
87
|
+
new_vars = set(bound_vars)
|
|
88
|
+
self.function_params[param_name] = sorted(existing | new_vars)
|
|
89
|
+
|
|
90
|
+
def get_param_for_expr(self, block_idx: int, expr: ast.AST) -> Optional[str]:
|
|
91
|
+
"""Get the parameter name for an expression."""
|
|
92
|
+
expr_str = ast.unparse(expr)
|
|
93
|
+
return self.mappings.get((block_idx, expr_str))
|
|
94
|
+
|
|
95
|
+
def is_function_param(self, param_name: str) -> bool:
|
|
96
|
+
"""Check if a parameter should be a function parameter."""
|
|
97
|
+
return param_name in self.function_params
|
|
98
|
+
|
|
99
|
+
def get_function_param_vars(self, param_name: str) -> List[str]:
|
|
100
|
+
"""Get the bound variables a function parameter should take."""
|
|
101
|
+
return self.function_params.get(param_name, [])
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def get_free_variables(expr: ast.AST) -> Set[str]:
|
|
105
|
+
"""
|
|
106
|
+
Get all variables referenced in an expression (Load context only).
|
|
107
|
+
|
|
108
|
+
Args:
|
|
109
|
+
expr: Expression AST node
|
|
110
|
+
|
|
111
|
+
Returns:
|
|
112
|
+
Set of variable names referenced in the expression
|
|
113
|
+
"""
|
|
114
|
+
|
|
115
|
+
class VarCollector(ast.NodeVisitor):
|
|
116
|
+
def __init__(self) -> None:
|
|
117
|
+
self.vars: Set[str] = set()
|
|
118
|
+
|
|
119
|
+
def visit_Name(self, node: ast.Name) -> None:
|
|
120
|
+
if isinstance(node.ctx, ast.Load):
|
|
121
|
+
self.vars.add(node.id)
|
|
122
|
+
self.generic_visit(node)
|
|
123
|
+
|
|
124
|
+
collector = VarCollector()
|
|
125
|
+
collector.visit(expr)
|
|
126
|
+
return collector.vars
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def get_bound_variables_in_context(node: ast.AST, target_expr: ast.AST) -> Set[str]:
|
|
130
|
+
"""
|
|
131
|
+
Get variables that are bound in the context surrounding target_expr within node.
|
|
132
|
+
|
|
133
|
+
This includes variables bound by:
|
|
134
|
+
1. Enclosing for loops: 'for item in items: ... item ...'
|
|
135
|
+
2. Enclosing comprehensions: '[item for item in items]'
|
|
136
|
+
3. Enclosing lambdas: 'lambda x: x + 1'
|
|
137
|
+
4. Assignments anywhere in the block: 'cleaned = x.strip(); ... cleaned ...'
|
|
138
|
+
|
|
139
|
+
For assignments, we use a simpler rule: if a variable is assigned anywhere in the
|
|
140
|
+
block containing the expression, it's considered bound. This matches Python's
|
|
141
|
+
scoping rules where assignment anywhere in a function makes a variable local.
|
|
142
|
+
|
|
143
|
+
Args:
|
|
144
|
+
node: Root AST node to search in
|
|
145
|
+
target_expr: Expression we're looking for
|
|
146
|
+
|
|
147
|
+
Returns:
|
|
148
|
+
Set of variables bound in context surrounding target_expr
|
|
149
|
+
"""
|
|
150
|
+
|
|
151
|
+
# Find bound variables by traversing the AST with a stack
|
|
152
|
+
class BindingContextFinder(ast.NodeVisitor):
|
|
153
|
+
def __init__(self, target: ast.AST) -> None:
|
|
154
|
+
self.target: ast.AST = target
|
|
155
|
+
self.target_str: str = ast.unparse(target)
|
|
156
|
+
self.bound_vars: Set[str] = set()
|
|
157
|
+
self.found_target: bool = False
|
|
158
|
+
# Stack of currently bound variables (for control structures)
|
|
159
|
+
self.binding_stack: List[Set[str]] = []
|
|
160
|
+
# Accumulated assignments (persist for rest of block)
|
|
161
|
+
self.assignments: Set[str] = set()
|
|
162
|
+
|
|
163
|
+
def _contains_target(self, node: ast.AST) -> bool:
|
|
164
|
+
"""Check if node contains the target expression."""
|
|
165
|
+
return self.target_str in ast.unparse(node)
|
|
166
|
+
|
|
167
|
+
def _get_binding_vars(self, target: ast.AST) -> Set[str]:
|
|
168
|
+
"""Extract variable names from a binding target (Name, Tuple, etc.)."""
|
|
169
|
+
if isinstance(target, ast.Name):
|
|
170
|
+
return {target.id}
|
|
171
|
+
elif isinstance(target, (ast.Tuple, ast.List)):
|
|
172
|
+
vars = set()
|
|
173
|
+
for elt in target.elts:
|
|
174
|
+
vars.update(self._get_binding_vars(elt))
|
|
175
|
+
return vars
|
|
176
|
+
else:
|
|
177
|
+
return set()
|
|
178
|
+
|
|
179
|
+
def visit_For(self, node: ast.For) -> None:
|
|
180
|
+
if self._contains_target(node):
|
|
181
|
+
self._visit_binding_target(node, lambda: self.generic_visit(node))
|
|
182
|
+
else:
|
|
183
|
+
self.generic_visit(node)
|
|
184
|
+
|
|
185
|
+
def visit_comprehension(self, node: ast.comprehension) -> None:
|
|
186
|
+
# comprehension node (part of generators list in ListComp, etc.)
|
|
187
|
+
if self._contains_target(node):
|
|
188
|
+
self._visit_binding_target(node, lambda: self.generic_visit(node))
|
|
189
|
+
else:
|
|
190
|
+
self.generic_visit(node)
|
|
191
|
+
|
|
192
|
+
def visit_ListComp(self, node: ast.ListComp) -> None:
|
|
193
|
+
# CRITICAL: Comprehension variables must be bound when visiting elt
|
|
194
|
+
# [r.get_value() for r in results] - 'r' must be bound before visiting r.get_value()
|
|
195
|
+
if self._contains_target(node):
|
|
196
|
+
self._visit_comprehension_node(node, lambda: self.visit(node.elt))
|
|
197
|
+
else:
|
|
198
|
+
self.generic_visit(node)
|
|
199
|
+
|
|
200
|
+
def visit_SetComp(self, node: ast.SetComp) -> None:
|
|
201
|
+
# CRITICAL: Comprehension variables must be bound when visiting elt
|
|
202
|
+
if self._contains_target(node):
|
|
203
|
+
self._visit_comprehension_node(node, lambda: self.visit(node.elt))
|
|
204
|
+
else:
|
|
205
|
+
self.generic_visit(node)
|
|
206
|
+
|
|
207
|
+
def visit_DictComp(self, node: ast.DictComp) -> None:
|
|
208
|
+
# CRITICAL: Comprehension variables must be bound when visiting key and value
|
|
209
|
+
if self._contains_target(node):
|
|
210
|
+
|
|
211
|
+
def visit_entries() -> None:
|
|
212
|
+
self.visit(node.key)
|
|
213
|
+
self.visit(node.value)
|
|
214
|
+
|
|
215
|
+
self._visit_comprehension_node(node, visit_entries)
|
|
216
|
+
else:
|
|
217
|
+
self.generic_visit(node)
|
|
218
|
+
|
|
219
|
+
def visit_GeneratorExp(self, node: ast.GeneratorExp) -> None:
|
|
220
|
+
# CRITICAL: Comprehension variables must be bound when visiting elt
|
|
221
|
+
if self._contains_target(node):
|
|
222
|
+
self._visit_comprehension_node(node, lambda: self.visit(node.elt))
|
|
223
|
+
else:
|
|
224
|
+
self.generic_visit(node)
|
|
225
|
+
|
|
226
|
+
def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
|
|
227
|
+
# Function creates a new scope - save current assignments and start fresh
|
|
228
|
+
self._visit_function_like(node)
|
|
229
|
+
|
|
230
|
+
def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
|
|
231
|
+
# Same as FunctionDef
|
|
232
|
+
# Async functions mirror FunctionDef handling but use AsyncFunctionDef fields
|
|
233
|
+
self._visit_function_like(node)
|
|
234
|
+
|
|
235
|
+
def visit_ClassDef(self, node: ast.ClassDef) -> None:
|
|
236
|
+
# Class creates a new scope - save current assignments and start fresh
|
|
237
|
+
if self._contains_target(node):
|
|
238
|
+
self._visit_class_scope(node)
|
|
239
|
+
else:
|
|
240
|
+
self.assignments.add(node.name)
|
|
241
|
+
|
|
242
|
+
def visit_Lambda(self, node: ast.Lambda) -> None:
|
|
243
|
+
# Lambda creates a new scope for its parameters
|
|
244
|
+
if self._contains_target(node):
|
|
245
|
+
lambda_vars = {arg.arg for arg in node.args.args}
|
|
246
|
+
if lambda_vars:
|
|
247
|
+
self.binding_stack.append(lambda_vars)
|
|
248
|
+
self.visit(node.body)
|
|
249
|
+
if lambda_vars:
|
|
250
|
+
self.binding_stack.pop()
|
|
251
|
+
|
|
252
|
+
def visit_Assign(self, node: ast.Assign) -> None:
|
|
253
|
+
# CRITICAL: Assignments create bindings that persist for the rest of the block
|
|
254
|
+
# We accumulate ALL assignments as we traverse (not just those containing target)
|
|
255
|
+
# Extract assigned variable(s)
|
|
256
|
+
for target in node.targets:
|
|
257
|
+
self.assignments.update(self._get_binding_vars(target))
|
|
258
|
+
self.generic_visit(node)
|
|
259
|
+
|
|
260
|
+
def visit_AugAssign(self, node: ast.AugAssign) -> None:
|
|
261
|
+
# Augmented assignments also create persistent bindings
|
|
262
|
+
self.assignments.update(self._get_binding_vars(node.target))
|
|
263
|
+
self.generic_visit(node)
|
|
264
|
+
|
|
265
|
+
def visit_AnnAssign(self, node: ast.AnnAssign) -> None:
|
|
266
|
+
# Annotated assignments create persistent bindings
|
|
267
|
+
self.assignments.update(self._get_binding_vars(node.target))
|
|
268
|
+
self.generic_visit(node)
|
|
269
|
+
|
|
270
|
+
def visit_With(self, node: ast.With) -> None:
|
|
271
|
+
# Handle with statements: with open(f) as file: ...
|
|
272
|
+
if self._contains_target(node):
|
|
273
|
+
with_vars: Set[str] = set()
|
|
274
|
+
for item in node.items:
|
|
275
|
+
if item.optional_vars:
|
|
276
|
+
with_vars.update(self._get_binding_vars(item.optional_vars))
|
|
277
|
+
if with_vars:
|
|
278
|
+
self._with_binding(with_vars, lambda: self.generic_visit(node))
|
|
279
|
+
else:
|
|
280
|
+
self.generic_visit(node)
|
|
281
|
+
else:
|
|
282
|
+
self.generic_visit(node)
|
|
283
|
+
|
|
284
|
+
def visit_ExceptHandler(self, node: ast.ExceptHandler) -> None:
|
|
285
|
+
# Handle exception handlers: except Exception as e: ...
|
|
286
|
+
if self._contains_target(node):
|
|
287
|
+
if node.name:
|
|
288
|
+
# Exception variable is bound
|
|
289
|
+
self.binding_stack.append({node.name})
|
|
290
|
+
self.generic_visit(node)
|
|
291
|
+
self.binding_stack.pop()
|
|
292
|
+
else:
|
|
293
|
+
self.generic_visit(node)
|
|
294
|
+
else:
|
|
295
|
+
self.generic_visit(node)
|
|
296
|
+
|
|
297
|
+
def visit_NamedExpr(self, node: ast.NamedExpr) -> None:
|
|
298
|
+
# Handle walrus operator: if (x := foo()): ...
|
|
299
|
+
if self._contains_target(node):
|
|
300
|
+
# The target of := is a binding
|
|
301
|
+
named_vars = self._get_binding_vars(node.target)
|
|
302
|
+
self._with_binding(named_vars, lambda: self.generic_visit(node))
|
|
303
|
+
else:
|
|
304
|
+
self.generic_visit(node)
|
|
305
|
+
|
|
306
|
+
def generic_visit(self, node: ast.AST) -> None:
|
|
307
|
+
# Check if this node matches target
|
|
308
|
+
if ast.unparse(node) == self.target_str:
|
|
309
|
+
self.found_target = True
|
|
310
|
+
# Collect all currently bound variables
|
|
311
|
+
# (from both control structures and assignments)
|
|
312
|
+
for bound_set in self.binding_stack:
|
|
313
|
+
self.bound_vars.update(bound_set)
|
|
314
|
+
self.bound_vars.update(self.assignments)
|
|
315
|
+
ast.NodeVisitor.generic_visit(self, node)
|
|
316
|
+
|
|
317
|
+
def _with_binding(self, names: Set[str], visit: Callable[[], None]) -> None:
|
|
318
|
+
if not names:
|
|
319
|
+
visit()
|
|
320
|
+
return
|
|
321
|
+
self.binding_stack.append(names)
|
|
322
|
+
try:
|
|
323
|
+
visit()
|
|
324
|
+
finally:
|
|
325
|
+
self.binding_stack.pop()
|
|
326
|
+
|
|
327
|
+
def _visit_binding_target(
|
|
328
|
+
self, node: Union[ast.For, ast.comprehension], visit: Callable[[], None]
|
|
329
|
+
) -> None:
|
|
330
|
+
loop_vars = self._get_binding_vars(node.target)
|
|
331
|
+
self._with_binding(loop_vars, visit)
|
|
332
|
+
|
|
333
|
+
def _visit_comprehension_node(
|
|
334
|
+
self,
|
|
335
|
+
node: Union[ast.ListComp, ast.DictComp, ast.SetComp, ast.GeneratorExp],
|
|
336
|
+
visit_expression: Callable[[], None],
|
|
337
|
+
) -> None:
|
|
338
|
+
def _visit_generators() -> None:
|
|
339
|
+
for gen in node.generators:
|
|
340
|
+
self.visit(gen.iter)
|
|
341
|
+
for if_clause in gen.ifs:
|
|
342
|
+
self.visit(if_clause)
|
|
343
|
+
|
|
344
|
+
for gen in node.generators:
|
|
345
|
+
comp_vars = self._get_binding_vars(gen.target)
|
|
346
|
+
self.binding_stack.append(comp_vars)
|
|
347
|
+
|
|
348
|
+
try:
|
|
349
|
+
_visit_generators()
|
|
350
|
+
visit_expression()
|
|
351
|
+
finally:
|
|
352
|
+
for _ in node.generators:
|
|
353
|
+
self.binding_stack.pop()
|
|
354
|
+
|
|
355
|
+
def _collect_param_names(self, args: ast.arguments) -> Set[str]:
|
|
356
|
+
names: Set[str] = set()
|
|
357
|
+
for arg in args.args:
|
|
358
|
+
names.add(arg.arg)
|
|
359
|
+
for arg in args.posonlyargs:
|
|
360
|
+
names.add(arg.arg)
|
|
361
|
+
for arg in args.kwonlyargs:
|
|
362
|
+
names.add(arg.arg)
|
|
363
|
+
if args.vararg:
|
|
364
|
+
names.add(args.vararg.arg)
|
|
365
|
+
if args.kwarg:
|
|
366
|
+
names.add(args.kwarg.arg)
|
|
367
|
+
return names
|
|
368
|
+
|
|
369
|
+
def _visit_function_like(self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef]) -> None:
|
|
370
|
+
if not self._contains_target(node):
|
|
371
|
+
self.assignments.add(node.name)
|
|
372
|
+
return
|
|
373
|
+
|
|
374
|
+
saved_assignments = self.assignments.copy()
|
|
375
|
+
self.assignments.add(node.name)
|
|
376
|
+
self.binding_stack.append({node.name})
|
|
377
|
+
try:
|
|
378
|
+
param_names = self._collect_param_names(node.args)
|
|
379
|
+
if param_names:
|
|
380
|
+
self._with_binding(
|
|
381
|
+
param_names,
|
|
382
|
+
lambda: self._visit_function_body(node, saved_assignments),
|
|
383
|
+
)
|
|
384
|
+
else:
|
|
385
|
+
self._visit_function_body(node, saved_assignments)
|
|
386
|
+
finally:
|
|
387
|
+
self.binding_stack.pop()
|
|
388
|
+
|
|
389
|
+
def _visit_function_body(
|
|
390
|
+
self, node: Union[ast.FunctionDef, ast.AsyncFunctionDef], saved_assignments: Set[str]
|
|
391
|
+
) -> None:
|
|
392
|
+
self.assignments = set()
|
|
393
|
+
try:
|
|
394
|
+
for stmt in node.body:
|
|
395
|
+
self.visit(stmt)
|
|
396
|
+
finally:
|
|
397
|
+
self.assignments = saved_assignments
|
|
398
|
+
|
|
399
|
+
def _visit_class_scope(self, node: ast.ClassDef) -> None:
|
|
400
|
+
saved_assignments = self.assignments.copy()
|
|
401
|
+
self.assignments.add(node.name)
|
|
402
|
+
self.binding_stack.append({node.name})
|
|
403
|
+
try:
|
|
404
|
+
self.assignments = set()
|
|
405
|
+
for stmt in node.body:
|
|
406
|
+
self.visit(stmt)
|
|
407
|
+
finally:
|
|
408
|
+
self.assignments = saved_assignments
|
|
409
|
+
self.binding_stack.pop()
|
|
410
|
+
|
|
411
|
+
finder = BindingContextFinder(target_expr)
|
|
412
|
+
finder.visit(node)
|
|
413
|
+
|
|
414
|
+
# Filter to only include variables that are actually referenced in the target expression
|
|
415
|
+
vars_in_expr = get_free_variables(target_expr)
|
|
416
|
+
result = finder.bound_vars & vars_in_expr
|
|
417
|
+
|
|
418
|
+
return result
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
class Unifier:
|
|
422
|
+
"""
|
|
423
|
+
Unify AST blocks to find parameterizable differences.
|
|
424
|
+
|
|
425
|
+
This finds sub-expressions that differ between blocks and can be
|
|
426
|
+
factored out into function parameters.
|
|
427
|
+
|
|
428
|
+
Implements alpha-renaming for bound variables (loop vars, etc.).
|
|
429
|
+
"""
|
|
430
|
+
|
|
431
|
+
# Track constant occurrences: (block_idx, value) -> [position_paths]
|
|
432
|
+
# position_path is a tuple of (stmt_idx, field_name, ...) identifying location
|
|
433
|
+
constant_positions: Dict[Tuple[int, Any], List[Tuple[Any, ...]]]
|
|
434
|
+
|
|
435
|
+
def __init__(
|
|
436
|
+
self,
|
|
437
|
+
max_parameters: int = 5,
|
|
438
|
+
parameterize_constants: bool = True,
|
|
439
|
+
*,
|
|
440
|
+
promote_equal_hof_literals: bool = False,
|
|
441
|
+
):
|
|
442
|
+
"""
|
|
443
|
+
Initialize unifier.
|
|
444
|
+
|
|
445
|
+
Args:
|
|
446
|
+
max_parameters: Maximum number of parameters to extract
|
|
447
|
+
parameterize_constants: Whether to parameterize differing constants
|
|
448
|
+
"""
|
|
449
|
+
self.max_parameters = max_parameters
|
|
450
|
+
# Feature flag: when True, enable Option B promotion of equal literals in
|
|
451
|
+
# higher-order factory calls (thread as parameters even when equal).
|
|
452
|
+
self._set_feature_flags(parameterize_constants, promote_equal_hof_literals)
|
|
453
|
+
self.param_counter = 0
|
|
454
|
+
# Track alpha-equivalence mappings for bound variables
|
|
455
|
+
# Maps (block_idx, original_name) -> canonical_name
|
|
456
|
+
self.alpha_renamings: Dict[Tuple[int, str], str] = {}
|
|
457
|
+
# Store blocks being unified for context analysis
|
|
458
|
+
self.current_blocks: Optional[List[List[ast.AST]]] = None
|
|
459
|
+
# Track constant occurrences: (block_idx, value) -> [position_paths]
|
|
460
|
+
# position_path is a tuple of (stmt_idx, field_name, ...) identifying location
|
|
461
|
+
self.constant_positions = {}
|
|
462
|
+
|
|
463
|
+
def unify_blocks(
|
|
464
|
+
self, blocks: List[List[ast.AST]], hygienic_renames: List[Dict[str, str]]
|
|
465
|
+
) -> Optional[Substitution]:
|
|
466
|
+
"""
|
|
467
|
+
Unify multiple code blocks.
|
|
468
|
+
|
|
469
|
+
Args:
|
|
470
|
+
blocks: List of code blocks (each is a list of AST statements)
|
|
471
|
+
hygienic_renames: For each block, a mapping from original names
|
|
472
|
+
to hygienically renamed names
|
|
473
|
+
|
|
474
|
+
Returns:
|
|
475
|
+
Substitution mapping expressions to parameters, or None if unification fails
|
|
476
|
+
"""
|
|
477
|
+
if len(blocks) < 2:
|
|
478
|
+
return None
|
|
479
|
+
|
|
480
|
+
# Check all blocks have the same number of statements
|
|
481
|
+
if not all(len(b) == len(blocks[0]) for b in blocks):
|
|
482
|
+
return None
|
|
483
|
+
|
|
484
|
+
# Reset per-unification state to avoid cross-pair contamination
|
|
485
|
+
# Alpha-renamings and parameter counters must start fresh for each call
|
|
486
|
+
self.alpha_renamings = {}
|
|
487
|
+
self._reset_unification_state(blocks)
|
|
488
|
+
|
|
489
|
+
# Collect all constant positions for consistency checking
|
|
490
|
+
self._collect_constant_positions(blocks)
|
|
491
|
+
|
|
492
|
+
# Detect and map block-level bound variables for hygienic renaming
|
|
493
|
+
# This allows unification of blocks with structurally identical code but different variable names
|
|
494
|
+
self._setup_bound_variable_alpha_renamings(blocks)
|
|
495
|
+
|
|
496
|
+
# Initialize substitution
|
|
497
|
+
subst = Substitution()
|
|
498
|
+
|
|
499
|
+
# Unify statement by statement
|
|
500
|
+
for stmt_idx in range(len(blocks[0])):
|
|
501
|
+
stmts = [block[stmt_idx] for block in blocks]
|
|
502
|
+
|
|
503
|
+
# Unify this statement across all blocks
|
|
504
|
+
if not self._unify_nodes(stmts, subst, list(range(len(blocks)))):
|
|
505
|
+
return None
|
|
506
|
+
|
|
507
|
+
# Check we haven't exceeded max parameters
|
|
508
|
+
if len(subst.param_expressions) > self.max_parameters:
|
|
509
|
+
return None
|
|
510
|
+
|
|
511
|
+
# Copy alpha-renamings to output hygienic_renames parameter
|
|
512
|
+
# This allows callers to access the renames that were applied
|
|
513
|
+
for (block_idx, var_name), canonical_name in self.alpha_renamings.items():
|
|
514
|
+
if block_idx < len(hygienic_renames):
|
|
515
|
+
hygienic_renames[block_idx][var_name] = canonical_name
|
|
516
|
+
|
|
517
|
+
# Also attach hygienic_renames to the substitution for downstream consumers
|
|
518
|
+
# so they don't need to thread the mapping through every call.
|
|
519
|
+
try:
|
|
520
|
+
subst.hygienic_renames = hygienic_renames
|
|
521
|
+
except Exception:
|
|
522
|
+
# Best-effort; continue even if attribute assignment is blocked
|
|
523
|
+
pass
|
|
524
|
+
|
|
525
|
+
# After successful unification, optionally promote literal arguments in
|
|
526
|
+
# higher-order factory calls (Option B policy): even if literals are
|
|
527
|
+
# equal across blocks, expose them as parameters and thread through calls.
|
|
528
|
+
if self.promote_equal_hof_literals:
|
|
529
|
+
try:
|
|
530
|
+
self._promote_hof_literals(blocks, subst)
|
|
531
|
+
except Exception:
|
|
532
|
+
# Non-fatal; promotion is best-effort
|
|
533
|
+
pass
|
|
534
|
+
|
|
535
|
+
return subst
|
|
536
|
+
|
|
537
|
+
def _promote_hof_literals(self, blocks: List[List[ast.AST]], subst: Substitution) -> None:
|
|
538
|
+
"""
|
|
539
|
+
Promote literal arguments in higher-order factory calls into parameters,
|
|
540
|
+
even when equal across blocks (Option B).
|
|
541
|
+
|
|
542
|
+
Heuristic:
|
|
543
|
+
- Look for assignments of the form: name = Call(...)
|
|
544
|
+
- If 'name' is later used in Call position (i.e., as a callee), treat the
|
|
545
|
+
assignment as constructing a higher-order function.
|
|
546
|
+
- For each Constant argument to that Call, promote it to a parameter by
|
|
547
|
+
mapping the corresponding per-block expression to a fresh parameter name.
|
|
548
|
+
|
|
549
|
+
Implementation notes:
|
|
550
|
+
- Uses block 0 as the structural template and assumes unified blocks share
|
|
551
|
+
the same AST shape for corresponding statements.
|
|
552
|
+
- Skips any expression already parameterized by the unifier.
|
|
553
|
+
- Generates fresh parameter names via self.param_counter to avoid collisions
|
|
554
|
+
with existing unified parameters (names like __param_N).
|
|
555
|
+
"""
|
|
556
|
+
|
|
557
|
+
if not blocks or len(blocks) < 2:
|
|
558
|
+
return
|
|
559
|
+
|
|
560
|
+
num_blocks = len(blocks)
|
|
561
|
+
|
|
562
|
+
# Utility: collect whether a variable is used later in a Call context (as a callee or as an argument)
|
|
563
|
+
def is_used_as_callable_or_value_later(
|
|
564
|
+
block: List[ast.AST], start_stmt_idx: int, var_name: str
|
|
565
|
+
) -> bool:
|
|
566
|
+
class CallContextFinder(ast.NodeVisitor):
|
|
567
|
+
def __init__(self) -> None:
|
|
568
|
+
self.found = False
|
|
569
|
+
|
|
570
|
+
def visit_Call(self, node: ast.Call) -> None:
|
|
571
|
+
# Used as a callee
|
|
572
|
+
if isinstance(node.func, ast.Name) and node.func.id == var_name:
|
|
573
|
+
self.found = True
|
|
574
|
+
# Used as an argument to a call (higher-order usage)
|
|
575
|
+
for arg in node.args:
|
|
576
|
+
if isinstance(arg, ast.Name) and arg.id == var_name:
|
|
577
|
+
self.found = True
|
|
578
|
+
for kw in node.keywords:
|
|
579
|
+
if (
|
|
580
|
+
kw.arg is not None
|
|
581
|
+
and isinstance(kw.value, ast.Name)
|
|
582
|
+
and kw.value.id == var_name
|
|
583
|
+
):
|
|
584
|
+
self.found = True
|
|
585
|
+
self.generic_visit(node)
|
|
586
|
+
|
|
587
|
+
def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
|
|
588
|
+
# Do not descend into nested function scopes
|
|
589
|
+
pass
|
|
590
|
+
|
|
591
|
+
def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
|
|
592
|
+
pass
|
|
593
|
+
|
|
594
|
+
def visit_ClassDef(self, node: ast.ClassDef) -> None:
|
|
595
|
+
pass
|
|
596
|
+
|
|
597
|
+
finder = CallContextFinder()
|
|
598
|
+
for sidx in range(start_stmt_idx + 1, len(block)):
|
|
599
|
+
finder.visit(block[sidx])
|
|
600
|
+
if finder.found:
|
|
601
|
+
return True
|
|
602
|
+
return False
|
|
603
|
+
|
|
604
|
+
# Utility: yield all (path, call_node) pairs within a statement in block 0
|
|
605
|
+
def iter_calls_with_paths(stmt: ast.AST) -> List[Tuple[Tuple[Any, ...], ast.Call]]:
|
|
606
|
+
result: List[Tuple[Tuple[Any, ...], ast.Call]] = []
|
|
607
|
+
|
|
608
|
+
def walk(node: ast.AST, path: Tuple[Any, ...]) -> None:
|
|
609
|
+
if isinstance(node, ast.Call):
|
|
610
|
+
result.append((path, node))
|
|
611
|
+
for field_name, value in self._iter_child_fields(node):
|
|
612
|
+
if isinstance(value, list):
|
|
613
|
+
for i, item in enumerate(value):
|
|
614
|
+
if isinstance(item, ast.AST):
|
|
615
|
+
walk(item, path + (field_name, i))
|
|
616
|
+
elif isinstance(value, ast.AST):
|
|
617
|
+
walk(value, path + (field_name,))
|
|
618
|
+
|
|
619
|
+
walk(stmt, ("$root",))
|
|
620
|
+
return result
|
|
621
|
+
|
|
622
|
+
# Utility: follow a path within a statement to retrieve the corresponding node
|
|
623
|
+
def get_node_by_path(stmt: ast.AST, path: Tuple[Any, ...]) -> Optional[ast.AST]:
|
|
624
|
+
node: ast.AST = stmt
|
|
625
|
+
# path starts with ("$root",), skip first marker
|
|
626
|
+
for p in path[1:]:
|
|
627
|
+
if isinstance(p, str):
|
|
628
|
+
if not hasattr(node, p):
|
|
629
|
+
return None
|
|
630
|
+
node = getattr(node, p)
|
|
631
|
+
elif isinstance(p, int):
|
|
632
|
+
# indexing into a list; prior element must have been a list field
|
|
633
|
+
# find the previous step to access the list; handled by caller structure
|
|
634
|
+
return None # we only use (field, index) pairs, so int alone shouldn't appear
|
|
635
|
+
else:
|
|
636
|
+
# we expect (field_name, index) pairs encoded sequentially
|
|
637
|
+
return None
|
|
638
|
+
return node
|
|
639
|
+
|
|
640
|
+
# Utility: get child by (field, index) sequence from current node
|
|
641
|
+
def get_node_by_field_index_path(stmt: ast.AST, path: Tuple[Any, ...]) -> Optional[ast.AST]:
|
|
642
|
+
node: ast.AST = stmt
|
|
643
|
+
# Skip "$root"
|
|
644
|
+
idx = 1
|
|
645
|
+
while idx < len(path):
|
|
646
|
+
part = path[idx]
|
|
647
|
+
if not isinstance(part, str):
|
|
648
|
+
return None
|
|
649
|
+
field = part
|
|
650
|
+
idx += 1
|
|
651
|
+
if idx < len(path) and isinstance(path[idx], int):
|
|
652
|
+
list_index = path[idx]
|
|
653
|
+
idx += 1
|
|
654
|
+
else:
|
|
655
|
+
list_index = None
|
|
656
|
+
|
|
657
|
+
value = getattr(node, field, None)
|
|
658
|
+
if list_index is None:
|
|
659
|
+
if not isinstance(value, ast.AST):
|
|
660
|
+
return None
|
|
661
|
+
node = value
|
|
662
|
+
else:
|
|
663
|
+
if not isinstance(value, list) or list_index >= len(value):
|
|
664
|
+
return None
|
|
665
|
+
next_node = value[list_index]
|
|
666
|
+
if not isinstance(next_node, ast.AST):
|
|
667
|
+
return None
|
|
668
|
+
node = next_node
|
|
669
|
+
return node
|
|
670
|
+
|
|
671
|
+
# Iterate over statements in block 0 and attempt promotions
|
|
672
|
+
for stmt_idx, stmt0 in enumerate(blocks[0]):
|
|
673
|
+
# Consider only simple assignments to a single Name
|
|
674
|
+
if not isinstance(stmt0, ast.Assign) or len(stmt0.targets) != 1:
|
|
675
|
+
continue
|
|
676
|
+
target0 = stmt0.targets[0]
|
|
677
|
+
if not isinstance(target0, ast.Name):
|
|
678
|
+
continue
|
|
679
|
+
if not isinstance(stmt0.value, ast.Call):
|
|
680
|
+
continue
|
|
681
|
+
|
|
682
|
+
target_name = target0.id
|
|
683
|
+
# Only promote when the assigned variable is used later in a call context
|
|
684
|
+
if not is_used_as_callable_or_value_later(blocks[0], stmt_idx, target_name):
|
|
685
|
+
continue
|
|
686
|
+
|
|
687
|
+
# For each arg in the call in block 0, if Constant, attempt to promote
|
|
688
|
+
call_paths = iter_calls_with_paths(stmt0)
|
|
689
|
+
# Find the specific call path corresponding to stmt0.value
|
|
690
|
+
# Since stmt0.value is a Call, find its path (should exist)
|
|
691
|
+
call_path = None
|
|
692
|
+
for path, call_node in call_paths:
|
|
693
|
+
if call_node is stmt0.value:
|
|
694
|
+
call_path = path
|
|
695
|
+
break
|
|
696
|
+
if call_path is None:
|
|
697
|
+
continue
|
|
698
|
+
|
|
699
|
+
call0 = stmt0.value
|
|
700
|
+
for arg_pos, arg0 in enumerate(call0.args):
|
|
701
|
+
if not isinstance(arg0, ast.AST):
|
|
702
|
+
continue
|
|
703
|
+
# Only consider literal constants for now
|
|
704
|
+
if not isinstance(arg0, ast.Constant):
|
|
705
|
+
continue
|
|
706
|
+
|
|
707
|
+
# Build path to this arg: call_path + ("args", arg_pos)
|
|
708
|
+
arg_path = call_path + ("args", arg_pos)
|
|
709
|
+
|
|
710
|
+
# Collect per-block corresponding arg expressions
|
|
711
|
+
per_block_exprs: List[Optional[ast.AST]] = []
|
|
712
|
+
missing = False
|
|
713
|
+
for bidx in range(num_blocks):
|
|
714
|
+
stmt_b = blocks[bidx][stmt_idx] if stmt_idx < len(blocks[bidx]) else None
|
|
715
|
+
if not isinstance(stmt_b, ast.Assign):
|
|
716
|
+
missing = True
|
|
717
|
+
break
|
|
718
|
+
val_b = stmt_b.value
|
|
719
|
+
if not isinstance(val_b, ast.AST):
|
|
720
|
+
missing = True
|
|
721
|
+
break
|
|
722
|
+
# Retrieve the node at arg_path within this statement
|
|
723
|
+
node_b = get_node_by_field_index_path(stmt_b, arg_path)
|
|
724
|
+
if node_b is None or not isinstance(node_b, ast.expr):
|
|
725
|
+
missing = True
|
|
726
|
+
break
|
|
727
|
+
per_block_exprs.append(node_b)
|
|
728
|
+
|
|
729
|
+
if missing or len(per_block_exprs) != num_blocks:
|
|
730
|
+
continue
|
|
731
|
+
|
|
732
|
+
# At this point, all per_block_exprs are non-None (validated above)
|
|
733
|
+
valid_exprs = cast(List[ast.AST], per_block_exprs)
|
|
734
|
+
|
|
735
|
+
# Skip if any of these expressions are already parameterized
|
|
736
|
+
already_param = False
|
|
737
|
+
for bidx, expr_b in enumerate(valid_exprs):
|
|
738
|
+
if subst.get_param_for_expr(bidx, expr_b) is not None:
|
|
739
|
+
already_param = True
|
|
740
|
+
break
|
|
741
|
+
if already_param:
|
|
742
|
+
continue
|
|
743
|
+
|
|
744
|
+
# Create fresh parameter name and record mappings for all blocks
|
|
745
|
+
param_name = f"__param_{self.param_counter}"
|
|
746
|
+
self.param_counter += 1
|
|
747
|
+
|
|
748
|
+
for bidx, expr_b in enumerate(valid_exprs):
|
|
749
|
+
subst.add_mapping(bidx, expr_b, param_name, bound_vars=None)
|
|
750
|
+
|
|
751
|
+
# Also store per-block expr under promoted_literal_args for clarity
|
|
752
|
+
if hasattr(subst, "promoted_literal_args"):
|
|
753
|
+
if param_name not in subst.promoted_literal_args:
|
|
754
|
+
subst.promoted_literal_args[param_name] = {}
|
|
755
|
+
for bidx, expr_b in enumerate(valid_exprs):
|
|
756
|
+
subst.promoted_literal_args[param_name][bidx] = expr_b
|
|
757
|
+
|
|
758
|
+
def _unify_nodes(
|
|
759
|
+
self, nodes: Sequence[ast.AST], subst: Substitution, block_indices: Sequence[int]
|
|
760
|
+
) -> bool:
|
|
761
|
+
"""
|
|
762
|
+
Unify a list of AST nodes (one from each block).
|
|
763
|
+
|
|
764
|
+
Args:
|
|
765
|
+
nodes: List of nodes to unify
|
|
766
|
+
subst: Current substitution
|
|
767
|
+
block_indices: Block index for each node
|
|
768
|
+
|
|
769
|
+
Returns:
|
|
770
|
+
True if unification succeeded
|
|
771
|
+
"""
|
|
772
|
+
# First check: do all nodes have the same type?
|
|
773
|
+
node_types = [type(n) for n in nodes]
|
|
774
|
+
if len(set(node_types)) != 1:
|
|
775
|
+
# Different types - cannot unify at the statement level
|
|
776
|
+
# Try to parameterize the entire expression
|
|
777
|
+
return self._try_parameterize(nodes, subst, block_indices)
|
|
778
|
+
|
|
779
|
+
first_node = nodes[0]
|
|
780
|
+
|
|
781
|
+
# Special handling for different node types
|
|
782
|
+
|
|
783
|
+
# Constants - check if they're identical, or parameterize if enabled
|
|
784
|
+
if isinstance(first_node, ast.Constant):
|
|
785
|
+
values = [cast(ast.Constant, n).value for n in nodes]
|
|
786
|
+
if len(set(values)) == 1:
|
|
787
|
+
return True # All same constant
|
|
788
|
+
|
|
789
|
+
# Different constants
|
|
790
|
+
if self.parameterize_constants:
|
|
791
|
+
# CRITICAL: Check consistency across all occurrences
|
|
792
|
+
# Per the user's rule: if a constant value appears at multiple positions,
|
|
793
|
+
# it must unify consistently at ALL positions.
|
|
794
|
+
#
|
|
795
|
+
# Example: "x = item * 2" vs "x = item * 3" AND "z = y ** 2" vs "z = y ** 2"
|
|
796
|
+
# The constant 2 appears at 2 positions in block 0
|
|
797
|
+
# Position 1: differs (2 vs 3)
|
|
798
|
+
# Position 2: same (2 vs 2)
|
|
799
|
+
# This is INCONSISTENT - we cannot parameterize just the constant 2
|
|
800
|
+
#
|
|
801
|
+
# Solution: Check if all occurrences of these values would unify consistently
|
|
802
|
+
|
|
803
|
+
if not self._check_constant_consistency(values, block_indices):
|
|
804
|
+
# Constants appear at multiple positions with inconsistent unification
|
|
805
|
+
# Cannot parameterize the bare constant
|
|
806
|
+
return False
|
|
807
|
+
|
|
808
|
+
# Parameterize differing constants
|
|
809
|
+
return self._try_parameterize(nodes, subst, block_indices)
|
|
810
|
+
else:
|
|
811
|
+
# Cannot unify - constants must be identical
|
|
812
|
+
return False
|
|
813
|
+
|
|
814
|
+
# Names - if they differ, check alpha-renaming first
|
|
815
|
+
if isinstance(first_node, ast.Name):
|
|
816
|
+
# Apply alpha-renaming to get canonical names
|
|
817
|
+
canonical_names = []
|
|
818
|
+
for idx, (node, block_idx) in enumerate(zip(nodes, block_indices)):
|
|
819
|
+
name = cast(ast.Name, node).id
|
|
820
|
+
# Check if this name has an alpha-renaming for this block
|
|
821
|
+
renamed = self.alpha_renamings.get((block_idx, name), name)
|
|
822
|
+
canonical_names.append(renamed)
|
|
823
|
+
|
|
824
|
+
# Check if all canonical names are the same
|
|
825
|
+
if len(set(canonical_names)) == 1:
|
|
826
|
+
return True # All same name (possibly after alpha-renaming)
|
|
827
|
+
|
|
828
|
+
# Different names even after alpha-renaming - track correspondence before parameterizing
|
|
829
|
+
# Use first ORIGINAL name (not canonical) for free variable correspondence
|
|
830
|
+
# This is important: we want to map admin→user, not admin→__temp_0
|
|
831
|
+
original_names = [cast(ast.Name, n).id for n in nodes]
|
|
832
|
+
first_original_name = original_names[0]
|
|
833
|
+
|
|
834
|
+
for node, block_idx, original_name in zip(nodes, block_indices, original_names):
|
|
835
|
+
if original_name != first_original_name:
|
|
836
|
+
# Record that this block's name maps to the first block's original name
|
|
837
|
+
# This handles free variables with different names across blocks
|
|
838
|
+
self.alpha_renamings[(block_idx, original_name)] = first_original_name
|
|
839
|
+
|
|
840
|
+
# Now parameterize
|
|
841
|
+
return self._try_parameterize(nodes, subst, block_indices)
|
|
842
|
+
|
|
843
|
+
# For compound nodes, recursively unify all fields
|
|
844
|
+
return self._unify_compound_node(nodes, subst, block_indices)
|
|
845
|
+
|
|
846
|
+
def _unify_compound_node(
|
|
847
|
+
self, nodes: Sequence[ast.AST], subst: Substitution, block_indices: Sequence[int]
|
|
848
|
+
) -> bool:
|
|
849
|
+
"""
|
|
850
|
+
Unify compound AST nodes by recursively unifying their fields.
|
|
851
|
+
|
|
852
|
+
Args:
|
|
853
|
+
nodes: List of nodes (all same type)
|
|
854
|
+
subst: Current substitution
|
|
855
|
+
block_indices: Block indices
|
|
856
|
+
|
|
857
|
+
Returns:
|
|
858
|
+
True if unification succeeded
|
|
859
|
+
"""
|
|
860
|
+
first_node = nodes[0]
|
|
861
|
+
|
|
862
|
+
# Special handling for binding constructs
|
|
863
|
+
# Loop variables, comprehension variables, etc. are BOUND by the construct
|
|
864
|
+
# If they differ (like 'i' vs 'j'), they're alpha-equivalent, not parameterizable
|
|
865
|
+
if isinstance(first_node, ast.For):
|
|
866
|
+
# For loops: the target variable is bound
|
|
867
|
+
# It can differ between blocks (like 'i' vs 'j') and that's OK
|
|
868
|
+
# We just need to unify the structure, not the variable name
|
|
869
|
+
return self._unify_for_loop(
|
|
870
|
+
cast(List[ast.For], nodes), subst, cast(List[int], list(block_indices))
|
|
871
|
+
)
|
|
872
|
+
|
|
873
|
+
# Special handling for Lambda: parameters are bindings (alpha-renaming)
|
|
874
|
+
# lambda x: x * 2 and lambda y: y * 2 are equivalent (alpha-equivalent)
|
|
875
|
+
# The parameter names should NOT be parameterized
|
|
876
|
+
if isinstance(first_node, ast.Lambda):
|
|
877
|
+
return self._unify_lambda(
|
|
878
|
+
cast(List[ast.Lambda], nodes), subst, cast(List[int], list(block_indices))
|
|
879
|
+
)
|
|
880
|
+
|
|
881
|
+
# Special handling for f-strings (JoinedStr)
|
|
882
|
+
# F-string literal parts (Constant nodes) must NEVER be parameterized
|
|
883
|
+
# Only the expressions inside FormattedValue can be parameterized
|
|
884
|
+
if isinstance(first_node, ast.JoinedStr):
|
|
885
|
+
return self._unify_joined_str(
|
|
886
|
+
cast(List[ast.JoinedStr], nodes), subst, cast(List[int], list(block_indices))
|
|
887
|
+
)
|
|
888
|
+
|
|
889
|
+
# Special handling for comprehensions: targets are bindings and may differ
|
|
890
|
+
# Treat generator targets as alpha-equivalent like for-loop variables
|
|
891
|
+
if isinstance(first_node, ast.ListComp):
|
|
892
|
+
return self._unify_list_comp(
|
|
893
|
+
cast(List[ast.ListComp], nodes), subst, cast(List[int], list(block_indices))
|
|
894
|
+
)
|
|
895
|
+
if isinstance(first_node, ast.SetComp):
|
|
896
|
+
return self._unify_set_comp(
|
|
897
|
+
cast(List[ast.SetComp], nodes), subst, cast(List[int], list(block_indices))
|
|
898
|
+
)
|
|
899
|
+
if isinstance(first_node, ast.DictComp):
|
|
900
|
+
return self._unify_dict_comp(
|
|
901
|
+
cast(List[ast.DictComp], nodes), subst, cast(List[int], list(block_indices))
|
|
902
|
+
)
|
|
903
|
+
if isinstance(first_node, ast.GeneratorExp):
|
|
904
|
+
return self._unify_generator_exp(
|
|
905
|
+
cast(List[ast.GeneratorExp], nodes), subst, cast(List[int], list(block_indices))
|
|
906
|
+
)
|
|
907
|
+
|
|
908
|
+
# Special handling for with-statements: optional_vars are bindings
|
|
909
|
+
if isinstance(first_node, ast.With):
|
|
910
|
+
return self._unify_with(
|
|
911
|
+
cast(List[ast.With], nodes), subst, cast(List[int], list(block_indices))
|
|
912
|
+
)
|
|
913
|
+
|
|
914
|
+
# Special handling for except handlers: name is a binding identifier
|
|
915
|
+
if isinstance(first_node, ast.ExceptHandler):
|
|
916
|
+
return self._unify_except_handler(
|
|
917
|
+
cast(List[ast.ExceptHandler], nodes), subst, cast(List[int], list(block_indices))
|
|
918
|
+
)
|
|
919
|
+
|
|
920
|
+
# Special handling for walrus operator: target is a binding (alpha-equivalent)
|
|
921
|
+
if isinstance(first_node, ast.NamedExpr):
|
|
922
|
+
return self._unify_named_expr(
|
|
923
|
+
cast(List[ast.NamedExpr], nodes), subst, cast(List[int], list(block_indices))
|
|
924
|
+
)
|
|
925
|
+
|
|
926
|
+
# For each field in the node
|
|
927
|
+
for field_name in first_node._fields:
|
|
928
|
+
# Skip location fields
|
|
929
|
+
if field_name in ("lineno", "col_offset", "end_lineno", "end_col_offset"):
|
|
930
|
+
continue
|
|
931
|
+
|
|
932
|
+
# Get field values from all nodes
|
|
933
|
+
field_values = [getattr(n, field_name, None) for n in nodes]
|
|
934
|
+
|
|
935
|
+
first_value = field_values[0]
|
|
936
|
+
|
|
937
|
+
# Handle different value types
|
|
938
|
+
if first_value is None:
|
|
939
|
+
# All None - OK
|
|
940
|
+
if not all(v is None for v in field_values):
|
|
941
|
+
# Some None, some not - can't unify
|
|
942
|
+
return False
|
|
943
|
+
continue
|
|
944
|
+
|
|
945
|
+
elif isinstance(first_value, list):
|
|
946
|
+
# Lists of AST nodes or primitives
|
|
947
|
+
# Check all are lists
|
|
948
|
+
if not all(isinstance(v, list) for v in field_values):
|
|
949
|
+
# Mixed list/non-list - can't unify
|
|
950
|
+
return False
|
|
951
|
+
if not self._unify_lists(
|
|
952
|
+
cast(Sequence[Sequence[Any]], field_values), subst, block_indices
|
|
953
|
+
):
|
|
954
|
+
return False
|
|
955
|
+
|
|
956
|
+
elif isinstance(first_value, ast.AST):
|
|
957
|
+
# Single AST node(s)
|
|
958
|
+
# Check all are AST nodes
|
|
959
|
+
if not all(isinstance(v, ast.AST) for v in field_values):
|
|
960
|
+
# Mixed AST/non-AST - can't unify
|
|
961
|
+
return False
|
|
962
|
+
|
|
963
|
+
# Special handling for operator nodes (no fields - just type markers)
|
|
964
|
+
# Operators like Add, Sub, Not, etc. define the operation semantics
|
|
965
|
+
# and should NOT be parameterized - they must be identical
|
|
966
|
+
if len(first_value._fields) == 0:
|
|
967
|
+
# Operator node - all must have same type
|
|
968
|
+
value_types = set(type(v) for v in field_values)
|
|
969
|
+
if len(value_types) > 1:
|
|
970
|
+
# Different operators - can't unify
|
|
971
|
+
return False
|
|
972
|
+
# Same operator type - continue
|
|
973
|
+
continue
|
|
974
|
+
|
|
975
|
+
# Regular AST nodes - _unify_nodes handles type differences via parameterization
|
|
976
|
+
if not self._unify_nodes(
|
|
977
|
+
cast(Sequence[ast.AST], field_values), subst, block_indices
|
|
978
|
+
):
|
|
979
|
+
return False
|
|
980
|
+
|
|
981
|
+
else:
|
|
982
|
+
# Primitive value (string, int, etc.)
|
|
983
|
+
# Check all are non-AST, non-list primitives
|
|
984
|
+
if any(isinstance(v, (ast.AST, list)) for v in field_values):
|
|
985
|
+
# Mixed primitive/AST or primitive/list - can't unify
|
|
986
|
+
return False
|
|
987
|
+
# Must all be equal
|
|
988
|
+
if not all(v == first_value for v in field_values):
|
|
989
|
+
# Try to parameterize the entire node if primitives differ
|
|
990
|
+
return self._try_parameterize(nodes, subst, block_indices)
|
|
991
|
+
|
|
992
|
+
return True
|
|
993
|
+
|
|
994
|
+
def _unify_list_comp(
|
|
995
|
+
self, nodes: List[ast.ListComp], subst: Substitution, block_indices: List[int]
|
|
996
|
+
) -> bool:
|
|
997
|
+
"""
|
|
998
|
+
Unify ListComp nodes with alpha-renaming of generator targets.
|
|
999
|
+
|
|
1000
|
+
For a list comprehension, variables bound in each generator's target are
|
|
1001
|
+
bindings (like for-loop variables) and can differ across blocks. We
|
|
1002
|
+
establish temporary alpha-renamings using the first block as canonical,
|
|
1003
|
+
unify all generators under those mappings, then unify the element.
|
|
1004
|
+
"""
|
|
1005
|
+
if not nodes:
|
|
1006
|
+
return False
|
|
1007
|
+
|
|
1008
|
+
return self._unify_comprehension_core(
|
|
1009
|
+
nodes,
|
|
1010
|
+
subst,
|
|
1011
|
+
block_indices,
|
|
1012
|
+
lambda: self._unify_nodes([n.elt for n in nodes], subst, block_indices),
|
|
1013
|
+
)
|
|
1014
|
+
|
|
1015
|
+
def _unify_set_comp(
|
|
1016
|
+
self, nodes: List[ast.SetComp], subst: Substitution, block_indices: List[int]
|
|
1017
|
+
) -> bool:
|
|
1018
|
+
if not nodes:
|
|
1019
|
+
return False
|
|
1020
|
+
|
|
1021
|
+
return self._unify_comprehension_core(
|
|
1022
|
+
nodes,
|
|
1023
|
+
subst,
|
|
1024
|
+
block_indices,
|
|
1025
|
+
lambda: self._unify_nodes([n.elt for n in nodes], subst, block_indices),
|
|
1026
|
+
)
|
|
1027
|
+
|
|
1028
|
+
def _unify_comprehension_core(
|
|
1029
|
+
self,
|
|
1030
|
+
nodes: Sequence[Union[ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp]],
|
|
1031
|
+
subst: Substitution,
|
|
1032
|
+
block_indices: List[int],
|
|
1033
|
+
finalize: Callable[[], bool],
|
|
1034
|
+
) -> bool:
|
|
1035
|
+
gen_lists = [n.generators for n in nodes]
|
|
1036
|
+
if not gen_lists or not all(len(g) == len(gen_lists[0]) for g in gen_lists):
|
|
1037
|
+
return False
|
|
1038
|
+
|
|
1039
|
+
saved_alpha = dict(self.alpha_renamings)
|
|
1040
|
+
try:
|
|
1041
|
+
num_gens = len(gen_lists[0])
|
|
1042
|
+
for gen_idx in range(num_gens):
|
|
1043
|
+
comps = [g[gen_idx] for g in gen_lists]
|
|
1044
|
+
if not self._unify_single_comprehension(comps, subst, block_indices):
|
|
1045
|
+
return False
|
|
1046
|
+
return finalize()
|
|
1047
|
+
finally:
|
|
1048
|
+
self.alpha_renamings = saved_alpha
|
|
1049
|
+
|
|
1050
|
+
def _unify_dict_comp(
|
|
1051
|
+
self, nodes: List[ast.DictComp], subst: Substitution, block_indices: List[int]
|
|
1052
|
+
) -> bool:
|
|
1053
|
+
if not nodes:
|
|
1054
|
+
return False
|
|
1055
|
+
|
|
1056
|
+
return self._unify_comprehension_core(
|
|
1057
|
+
nodes,
|
|
1058
|
+
subst,
|
|
1059
|
+
block_indices,
|
|
1060
|
+
lambda: self._unify_nodes([n.key for n in nodes], subst, block_indices)
|
|
1061
|
+
and self._unify_nodes([n.value for n in nodes], subst, block_indices),
|
|
1062
|
+
)
|
|
1063
|
+
|
|
1064
|
+
def _unify_generator_exp(
|
|
1065
|
+
self, nodes: List[ast.GeneratorExp], subst: Substitution, block_indices: List[int]
|
|
1066
|
+
) -> bool:
|
|
1067
|
+
if not nodes:
|
|
1068
|
+
return False
|
|
1069
|
+
|
|
1070
|
+
return self._unify_comprehension_core(
|
|
1071
|
+
nodes,
|
|
1072
|
+
subst,
|
|
1073
|
+
block_indices,
|
|
1074
|
+
lambda: self._unify_nodes([n.elt for n in nodes], subst, block_indices),
|
|
1075
|
+
)
|
|
1076
|
+
|
|
1077
|
+
def _unify_with(
|
|
1078
|
+
self, nodes: List[ast.With], subst: Substitution, block_indices: List[int]
|
|
1079
|
+
) -> bool:
|
|
1080
|
+
"""
|
|
1081
|
+
Unify With nodes, treating optional_vars as bound (alpha-equivalent).
|
|
1082
|
+
|
|
1083
|
+
- Unify items' context_expr.
|
|
1084
|
+
- Establish temporary alpha-renamings for optional_vars targets.
|
|
1085
|
+
- Unify bodies under those mappings.
|
|
1086
|
+
"""
|
|
1087
|
+
# Same number of items
|
|
1088
|
+
items_lists = [n.items for n in nodes]
|
|
1089
|
+
if not all(len(lst) == len(items_lists[0]) for lst in items_lists):
|
|
1090
|
+
return False
|
|
1091
|
+
|
|
1092
|
+
# Save mappings to restore after
|
|
1093
|
+
saved_alpha = dict(self.alpha_renamings)
|
|
1094
|
+
try:
|
|
1095
|
+
# Unify each item
|
|
1096
|
+
num_items = len(items_lists[0])
|
|
1097
|
+
for i in range(num_items):
|
|
1098
|
+
items_i = [lst[i] for lst in items_lists]
|
|
1099
|
+
# Unify context_expr
|
|
1100
|
+
if not self._unify_nodes([it.context_expr for it in items_i], subst, block_indices):
|
|
1101
|
+
return False
|
|
1102
|
+
|
|
1103
|
+
# Handle optional_vars as bindings
|
|
1104
|
+
optional_vars_raw = [it.optional_vars for it in items_i]
|
|
1105
|
+
if all(ov is None for ov in optional_vars_raw):
|
|
1106
|
+
pass # nothing to do
|
|
1107
|
+
else:
|
|
1108
|
+
# All must be AST; if any None while others not, fail
|
|
1109
|
+
if any(ov is None for ov in optional_vars_raw):
|
|
1110
|
+
return False
|
|
1111
|
+
# Establish alpha-renaming for targets
|
|
1112
|
+
targets = cast(List[ast.expr], optional_vars_raw)
|
|
1113
|
+
# Support simple Name or Tuple[List] of Names
|
|
1114
|
+
# Collect names positionally
|
|
1115
|
+
|
|
1116
|
+
def flatten_names(t: ast.AST) -> List[str]:
|
|
1117
|
+
if isinstance(t, ast.Name):
|
|
1118
|
+
return [t.id]
|
|
1119
|
+
if isinstance(t, (ast.Tuple, ast.List)):
|
|
1120
|
+
names: List[str] = []
|
|
1121
|
+
for e in t.elts:
|
|
1122
|
+
names.extend(flatten_names(e))
|
|
1123
|
+
return names
|
|
1124
|
+
return []
|
|
1125
|
+
|
|
1126
|
+
names_per_block = [flatten_names(t) for t in targets]
|
|
1127
|
+
# Ensure all have same arity
|
|
1128
|
+
arities = [len(nl) for nl in names_per_block]
|
|
1129
|
+
if len(set(arities)) != 1:
|
|
1130
|
+
return False
|
|
1131
|
+
# Use first block's names as canonical, map positionally
|
|
1132
|
+
for pos in range(arities[0]):
|
|
1133
|
+
canonical = names_per_block[0][pos]
|
|
1134
|
+
for idx, block_idx in enumerate(block_indices):
|
|
1135
|
+
actual = names_per_block[idx][pos]
|
|
1136
|
+
self.alpha_renamings[(block_idx, actual)] = canonical
|
|
1137
|
+
|
|
1138
|
+
# Unify bodies
|
|
1139
|
+
if not self._unify_lists([n.body for n in nodes], subst, block_indices):
|
|
1140
|
+
return False
|
|
1141
|
+
|
|
1142
|
+
# type_comment (if present) must match
|
|
1143
|
+
comments = [getattr(n, "type_comment", None) for n in nodes]
|
|
1144
|
+
if not all(c == comments[0] for c in comments):
|
|
1145
|
+
return False
|
|
1146
|
+
|
|
1147
|
+
return True
|
|
1148
|
+
finally:
|
|
1149
|
+
self.alpha_renamings = saved_alpha
|
|
1150
|
+
|
|
1151
|
+
def _unify_except_handler(
|
|
1152
|
+
self, nodes: List[ast.ExceptHandler], subst: Substitution, block_indices: List[int]
|
|
1153
|
+
) -> bool:
|
|
1154
|
+
"""
|
|
1155
|
+
Unify ExceptHandler nodes, treating the 'name' as a bound identifier.
|
|
1156
|
+
"""
|
|
1157
|
+
# Unify exception types (may be None)
|
|
1158
|
+
types_list: List[Optional[ast.expr]] = [n.type for n in nodes]
|
|
1159
|
+
if all(t is None for t in types_list):
|
|
1160
|
+
pass
|
|
1161
|
+
else:
|
|
1162
|
+
if any(t is None for t in types_list):
|
|
1163
|
+
return False
|
|
1164
|
+
if not self._unify_nodes(cast(List[ast.AST], types_list), subst, block_indices):
|
|
1165
|
+
return False
|
|
1166
|
+
|
|
1167
|
+
# Establish temporary alpha-renamings for the handler variable names (strings)
|
|
1168
|
+
saved_alpha = dict(self.alpha_renamings)
|
|
1169
|
+
try:
|
|
1170
|
+
names: List[Optional[str]] = [n.name for n in nodes]
|
|
1171
|
+
# If all None, fine; if some None and others not, fail
|
|
1172
|
+
if all(nm is None for nm in names):
|
|
1173
|
+
pass
|
|
1174
|
+
else:
|
|
1175
|
+
if not all((nm is None) == (names[0] is None) for nm in names):
|
|
1176
|
+
return False
|
|
1177
|
+
if names[0] is not None:
|
|
1178
|
+
names_str: List[str] = cast(List[str], names)
|
|
1179
|
+
canonical = names_str[0]
|
|
1180
|
+
for idx, block_idx in enumerate(block_indices):
|
|
1181
|
+
actual = names_str[idx]
|
|
1182
|
+
self.alpha_renamings[(block_idx, actual)] = canonical
|
|
1183
|
+
|
|
1184
|
+
# Unify body
|
|
1185
|
+
if not self._unify_lists([n.body for n in nodes], subst, block_indices):
|
|
1186
|
+
return False
|
|
1187
|
+
return True
|
|
1188
|
+
finally:
|
|
1189
|
+
self.alpha_renamings = saved_alpha
|
|
1190
|
+
|
|
1191
|
+
def _unify_named_expr(
|
|
1192
|
+
self, nodes: List[ast.NamedExpr], subst: Substitution, block_indices: List[int]
|
|
1193
|
+
) -> bool:
|
|
1194
|
+
"""
|
|
1195
|
+
Unify walrus (NamedExpr) treating the target as a binding.
|
|
1196
|
+
Only simple Name targets are recognized for alpha-renaming.
|
|
1197
|
+
"""
|
|
1198
|
+
# Save and set alpha mappings for targets
|
|
1199
|
+
saved_alpha = dict(self.alpha_renamings)
|
|
1200
|
+
try:
|
|
1201
|
+
targets = [n.target for n in nodes]
|
|
1202
|
+
name_targets: List[ast.Name] = [t for t in targets if isinstance(t, ast.Name)]
|
|
1203
|
+
if len(name_targets) == len(targets):
|
|
1204
|
+
names = [t.id for t in name_targets]
|
|
1205
|
+
canonical = names[0]
|
|
1206
|
+
for idx, block_idx in enumerate(block_indices):
|
|
1207
|
+
self.alpha_renamings[(block_idx, names[idx])] = canonical
|
|
1208
|
+
|
|
1209
|
+
# Unify values under established alpha-renamings
|
|
1210
|
+
if not self._unify_nodes([n.value for n in nodes], subst, block_indices):
|
|
1211
|
+
return False
|
|
1212
|
+
# Do not require exact match for target identifiers (treated as bindings)
|
|
1213
|
+
return True
|
|
1214
|
+
finally:
|
|
1215
|
+
self.alpha_renamings = saved_alpha
|
|
1216
|
+
|
|
1217
|
+
def _unify_single_comprehension(
|
|
1218
|
+
self, comps: List[ast.comprehension], subst: Substitution, block_indices: List[int]
|
|
1219
|
+
) -> bool:
|
|
1220
|
+
"""
|
|
1221
|
+
Unify a single 'comprehension' node across blocks, establishing
|
|
1222
|
+
alpha-renamings for its target (Name or Tuple of Names), then unifying
|
|
1223
|
+
its iterator and if-clauses under those mappings.
|
|
1224
|
+
"""
|
|
1225
|
+
# All must be comprehension nodes
|
|
1226
|
+
if not all(isinstance(c, ast.comprehension) for c in comps):
|
|
1227
|
+
return False
|
|
1228
|
+
|
|
1229
|
+
# is_async flags must match
|
|
1230
|
+
async_flags = [c.is_async for c in comps]
|
|
1231
|
+
if len(set(async_flags)) != 1:
|
|
1232
|
+
return False
|
|
1233
|
+
|
|
1234
|
+
# Handle targets as bindings
|
|
1235
|
+
targets = [c.target for c in comps]
|
|
1236
|
+
|
|
1237
|
+
# Simple Name targets
|
|
1238
|
+
if all(isinstance(t, ast.Name) for t in targets):
|
|
1239
|
+
name_targets = cast(List[ast.Name], targets)
|
|
1240
|
+
names = [t.id for t in name_targets]
|
|
1241
|
+
canonical = names[0]
|
|
1242
|
+
# Establish alpha-renaming for the duration of the entire comprehension
|
|
1243
|
+
for idx, block_idx in enumerate(block_indices):
|
|
1244
|
+
key = (block_idx, names[idx])
|
|
1245
|
+
self.alpha_renamings[key] = canonical
|
|
1246
|
+
|
|
1247
|
+
# Unify iterator and ifs under alpha-renaming
|
|
1248
|
+
if not self._unify_nodes([c.iter for c in comps], subst, block_indices):
|
|
1249
|
+
return False
|
|
1250
|
+
if not self._unify_lists([c.ifs for c in comps], subst, block_indices):
|
|
1251
|
+
return False
|
|
1252
|
+
return True
|
|
1253
|
+
|
|
1254
|
+
# Tuple targets with simple names
|
|
1255
|
+
if all(isinstance(t, ast.Tuple) for t in targets):
|
|
1256
|
+
# All tuples must be flat and same length with Name elts
|
|
1257
|
+
tuple_targets = cast(List[ast.Tuple], targets)
|
|
1258
|
+
lengths = [len(t.elts) for t in tuple_targets]
|
|
1259
|
+
if len(set(lengths)) != 1:
|
|
1260
|
+
return False
|
|
1261
|
+
if not all(all(isinstance(e, ast.Name) for e in t.elts) for t in tuple_targets):
|
|
1262
|
+
return False
|
|
1263
|
+
|
|
1264
|
+
tuple_names: List[List[str]] = [] # per position names
|
|
1265
|
+
for pos in range(lengths[0]):
|
|
1266
|
+
# We verified above that all elements are ast.Name, so cast for type checker
|
|
1267
|
+
tuple_names.append([cast(ast.Name, t.elts[pos]).id for t in tuple_targets])
|
|
1268
|
+
|
|
1269
|
+
# Establish alpha-renaming per position to the first block's names
|
|
1270
|
+
for pos, names_at_pos in enumerate(tuple_names):
|
|
1271
|
+
canonical = names_at_pos[0]
|
|
1272
|
+
for idx, block_idx in enumerate(block_indices):
|
|
1273
|
+
key = (block_idx, names_at_pos[idx])
|
|
1274
|
+
self.alpha_renamings[key] = canonical
|
|
1275
|
+
|
|
1276
|
+
# Unify iterator and ifs
|
|
1277
|
+
if not self._unify_nodes([c.iter for c in comps], subst, block_indices):
|
|
1278
|
+
return False
|
|
1279
|
+
if not self._unify_lists([c.ifs for c in comps], subst, block_indices):
|
|
1280
|
+
return False
|
|
1281
|
+
return True
|
|
1282
|
+
|
|
1283
|
+
# Fallback: complex targets - unify structurally without alpha-renaming
|
|
1284
|
+
return (
|
|
1285
|
+
self._unify_nodes([c.target for c in comps], subst, block_indices)
|
|
1286
|
+
and self._unify_nodes([c.iter for c in comps], subst, block_indices)
|
|
1287
|
+
and self._unify_lists([c.ifs for c in comps], subst, block_indices)
|
|
1288
|
+
)
|
|
1289
|
+
|
|
1290
|
+
def _unify_loop_components(
|
|
1291
|
+
self,
|
|
1292
|
+
nodes: Sequence[ast.For],
|
|
1293
|
+
subst: Substitution,
|
|
1294
|
+
block_indices: Sequence[int],
|
|
1295
|
+
) -> bool:
|
|
1296
|
+
return (
|
|
1297
|
+
self._unify_nodes([n.iter for n in nodes], subst, block_indices)
|
|
1298
|
+
and self._unify_lists([n.body for n in nodes], subst, block_indices)
|
|
1299
|
+
and self._unify_lists([n.orelse for n in nodes], subst, block_indices)
|
|
1300
|
+
)
|
|
1301
|
+
|
|
1302
|
+
def _unify_for_loop(
|
|
1303
|
+
self, nodes: List[ast.For], subst: Substitution, block_indices: List[int]
|
|
1304
|
+
) -> bool:
|
|
1305
|
+
"""
|
|
1306
|
+
Unify For loops, treating loop variables as bound (alpha-equivalent).
|
|
1307
|
+
|
|
1308
|
+
Args:
|
|
1309
|
+
nodes: List of For nodes
|
|
1310
|
+
subst: Current substitution
|
|
1311
|
+
block_indices: Block indices
|
|
1312
|
+
|
|
1313
|
+
Returns:
|
|
1314
|
+
True if unification succeeded
|
|
1315
|
+
"""
|
|
1316
|
+
# For loops have: target, iter, body, orelse
|
|
1317
|
+
# The 'target' is a bound variable - it can differ (i vs j) and that's OK
|
|
1318
|
+
|
|
1319
|
+
# Get the loop variable names
|
|
1320
|
+
targets = [n.target for n in nodes]
|
|
1321
|
+
|
|
1322
|
+
# Check if targets are simple Names or Tuples
|
|
1323
|
+
if not all(isinstance(t, ast.Name) for t in targets):
|
|
1324
|
+
# Check if all targets are tuples (for tuple unpacking support)
|
|
1325
|
+
if all(isinstance(t, ast.Tuple) for t in targets):
|
|
1326
|
+
# Delegate to tuple unpacking handler
|
|
1327
|
+
return self._unify_for_loop_with_tuple_targets(nodes, subst, block_indices)
|
|
1328
|
+
|
|
1329
|
+
# Complex targets (nested structures, etc.) - fall back to default unification
|
|
1330
|
+
return (
|
|
1331
|
+
self._unify_nodes(targets, subst, block_indices)
|
|
1332
|
+
and self._unify_nodes([n.iter for n in nodes], subst, block_indices)
|
|
1333
|
+
and self._unify_lists([n.body for n in nodes], subst, block_indices)
|
|
1334
|
+
and self._unify_lists([n.orelse for n in nodes], subst, block_indices)
|
|
1335
|
+
)
|
|
1336
|
+
|
|
1337
|
+
name_targets = cast(List[ast.Name], targets)
|
|
1338
|
+
loop_var_names = [t.id for t in name_targets]
|
|
1339
|
+
|
|
1340
|
+
# Check if all loop variables have the same name
|
|
1341
|
+
if len(set(loop_var_names)) == 1:
|
|
1342
|
+
# Same loop variable name - just unify normally
|
|
1343
|
+
return self._unify_loop_components(nodes, subst, block_indices)
|
|
1344
|
+
|
|
1345
|
+
# Different loop variable names (i vs j) - establish alpha-equivalence
|
|
1346
|
+
# Use the first block's variable name as canonical
|
|
1347
|
+
canonical_var = loop_var_names[0]
|
|
1348
|
+
|
|
1349
|
+
# Establish alpha-renaming mappings for all blocks
|
|
1350
|
+
# Save old mappings to restore later
|
|
1351
|
+
old_mappings: Dict[Tuple[int, str], str] = {}
|
|
1352
|
+
for idx, block_idx in enumerate(block_indices):
|
|
1353
|
+
var_name = loop_var_names[idx]
|
|
1354
|
+
key = (block_idx, var_name)
|
|
1355
|
+
self._assign_alpha_mapping(key, canonical_var, old_mappings)
|
|
1356
|
+
|
|
1357
|
+
try:
|
|
1358
|
+
# Unify the iterator
|
|
1359
|
+
return self._unify_loop_components(nodes, subst, block_indices)
|
|
1360
|
+
|
|
1361
|
+
finally:
|
|
1362
|
+
# Restore old mappings or remove new ones
|
|
1363
|
+
for idx, block_idx in enumerate(block_indices):
|
|
1364
|
+
var_name = loop_var_names[idx]
|
|
1365
|
+
key = (block_idx, var_name)
|
|
1366
|
+
self._restore_alpha_mapping(key, old_mappings)
|
|
1367
|
+
|
|
1368
|
+
def _unify_for_loop_with_tuple_targets(
|
|
1369
|
+
self, nodes: List[ast.For], subst: Substitution, block_indices: List[int]
|
|
1370
|
+
) -> bool:
|
|
1371
|
+
"""
|
|
1372
|
+
Unify For loops with tuple unpacking targets (e.g., for key, value in items).
|
|
1373
|
+
|
|
1374
|
+
Handles flat tuple unpacking with alpha-renaming:
|
|
1375
|
+
- for key, value in pairs
|
|
1376
|
+
- for k, v in pairs
|
|
1377
|
+
|
|
1378
|
+
Args:
|
|
1379
|
+
nodes: List of For nodes with Tuple targets
|
|
1380
|
+
subst: Current substitution
|
|
1381
|
+
block_indices: Block indices
|
|
1382
|
+
|
|
1383
|
+
Returns:
|
|
1384
|
+
True if unification succeeded
|
|
1385
|
+
"""
|
|
1386
|
+
targets = [n.target for n in nodes]
|
|
1387
|
+
|
|
1388
|
+
# Verify all targets are tuples
|
|
1389
|
+
if not all(isinstance(t, ast.Tuple) for t in targets):
|
|
1390
|
+
return False
|
|
1391
|
+
|
|
1392
|
+
# Check all tuples have the same number of elements
|
|
1393
|
+
tuple_targets = cast(List[ast.Tuple], targets)
|
|
1394
|
+
tuple_lengths = [len(t.elts) for t in tuple_targets]
|
|
1395
|
+
if len(set(tuple_lengths)) != 1:
|
|
1396
|
+
return False
|
|
1397
|
+
|
|
1398
|
+
# Check all tuple elements are simple Name nodes
|
|
1399
|
+
for target in tuple_targets:
|
|
1400
|
+
if not all(isinstance(elt, ast.Name) for elt in target.elts):
|
|
1401
|
+
return False
|
|
1402
|
+
|
|
1403
|
+
# Extract variable names for each position
|
|
1404
|
+
# var_names[position_idx] = [name_in_block0, name_in_block1, ...]
|
|
1405
|
+
num_positions = len(tuple_targets[0].elts)
|
|
1406
|
+
var_names = []
|
|
1407
|
+
for pos in range(num_positions):
|
|
1408
|
+
# We verified elements are ast.Name above; cast to satisfy type checker
|
|
1409
|
+
names_at_pos = [cast(ast.Name, target.elts[pos]).id for target in tuple_targets]
|
|
1410
|
+
var_names.append(names_at_pos)
|
|
1411
|
+
|
|
1412
|
+
# Check if all corresponding names are identical
|
|
1413
|
+
# If so, no alpha-renaming needed
|
|
1414
|
+
all_same = all(len(set(names)) == 1 for names in var_names)
|
|
1415
|
+
|
|
1416
|
+
if all_same:
|
|
1417
|
+
# All tuple unpacking uses same variable names - just unify normally
|
|
1418
|
+
return self._unify_loop_components(nodes, subst, block_indices)
|
|
1419
|
+
|
|
1420
|
+
# Different variable names - establish alpha-equivalence for each position
|
|
1421
|
+
# Use the first block's variable names as canonical
|
|
1422
|
+
canonical_vars = [names[0] for names in var_names]
|
|
1423
|
+
|
|
1424
|
+
# Establish alpha-renaming mappings for all positions and blocks
|
|
1425
|
+
# Save old mappings to restore later
|
|
1426
|
+
old_mappings: Dict[Tuple[int, str], str] = {}
|
|
1427
|
+
for pos_idx, canonical_var in enumerate(canonical_vars):
|
|
1428
|
+
for idx, block_idx in enumerate(block_indices):
|
|
1429
|
+
var_name = var_names[pos_idx][idx]
|
|
1430
|
+
key = (block_idx, var_name)
|
|
1431
|
+
self._assign_alpha_mapping(key, canonical_var, old_mappings)
|
|
1432
|
+
|
|
1433
|
+
try:
|
|
1434
|
+
# Unify the iterator
|
|
1435
|
+
return self._unify_loop_components(nodes, subst, block_indices)
|
|
1436
|
+
|
|
1437
|
+
finally:
|
|
1438
|
+
# Restore old mappings or remove new ones
|
|
1439
|
+
for pos_idx, canonical_var in enumerate(canonical_vars):
|
|
1440
|
+
for idx, block_idx in enumerate(block_indices):
|
|
1441
|
+
var_name = var_names[pos_idx][idx]
|
|
1442
|
+
key = (block_idx, var_name)
|
|
1443
|
+
self._restore_alpha_mapping(key, old_mappings)
|
|
1444
|
+
|
|
1445
|
+
def _unify_lambda(
|
|
1446
|
+
self, nodes: List[ast.Lambda], subst: Substitution, block_indices: List[int]
|
|
1447
|
+
) -> bool:
|
|
1448
|
+
"""
|
|
1449
|
+
Unify lambda expressions with alpha-renaming support.
|
|
1450
|
+
|
|
1451
|
+
Lambda parameters are bindings, similar to for loop variables.
|
|
1452
|
+
If they differ (like 'lambda x: ...' vs 'lambda y: ...'), they're
|
|
1453
|
+
alpha-equivalent, not parameterizable.
|
|
1454
|
+
|
|
1455
|
+
Args:
|
|
1456
|
+
nodes: List of Lambda nodes
|
|
1457
|
+
subst: Current substitution
|
|
1458
|
+
block_indices: Block indices
|
|
1459
|
+
|
|
1460
|
+
Returns:
|
|
1461
|
+
True if unification succeeded
|
|
1462
|
+
"""
|
|
1463
|
+
# For now, only handle simple case: same number of regular positional args
|
|
1464
|
+
# Get parameter counts for each lambda
|
|
1465
|
+
param_counts = [len(n.args.args) for n in nodes]
|
|
1466
|
+
if len(set(param_counts)) > 1:
|
|
1467
|
+
# Different number of parameters - can't unify
|
|
1468
|
+
return False
|
|
1469
|
+
|
|
1470
|
+
# Check that all other parameter types are empty (no *args, **kwargs, etc.)
|
|
1471
|
+
for node in nodes:
|
|
1472
|
+
if node.args.posonlyargs or node.args.kwonlyargs or node.args.vararg or node.args.kwarg:
|
|
1473
|
+
# Complex lambda parameters - not currently supported
|
|
1474
|
+
# See docs/KNOWN_LIMITATIONS.md for details
|
|
1475
|
+
return False
|
|
1476
|
+
|
|
1477
|
+
# Get parameter names from each lambda
|
|
1478
|
+
num_params = param_counts[0]
|
|
1479
|
+
if num_params == 0:
|
|
1480
|
+
# No parameters - just unify bodies directly
|
|
1481
|
+
return self._unify_nodes([n.body for n in nodes], subst, block_indices)
|
|
1482
|
+
|
|
1483
|
+
# Create alpha-renaming mappings for lambda parameters
|
|
1484
|
+
# Use first lambda's parameter names as canonical
|
|
1485
|
+
canonical_params = [nodes[0].args.args[i].arg for i in range(num_params)]
|
|
1486
|
+
|
|
1487
|
+
# Save old alpha-renaming mappings (in case of nested lambdas)
|
|
1488
|
+
old_mappings = {}
|
|
1489
|
+
try:
|
|
1490
|
+
# Set up alpha-renamings for each parameter position
|
|
1491
|
+
for param_idx in range(num_params):
|
|
1492
|
+
canonical_param = canonical_params[param_idx]
|
|
1493
|
+
for idx, node in enumerate(nodes):
|
|
1494
|
+
block_idx = block_indices[idx]
|
|
1495
|
+
actual_param = node.args.args[param_idx].arg
|
|
1496
|
+
key = (block_idx, actual_param)
|
|
1497
|
+
self._assign_alpha_mapping(key, canonical_param, old_mappings)
|
|
1498
|
+
|
|
1499
|
+
# Unify lambda bodies with alpha-renaming in effect
|
|
1500
|
+
return self._unify_nodes([n.body for n in nodes], subst, block_indices)
|
|
1501
|
+
|
|
1502
|
+
finally:
|
|
1503
|
+
# Restore old mappings or remove new ones
|
|
1504
|
+
for param_idx in range(num_params):
|
|
1505
|
+
for idx, node in enumerate(nodes):
|
|
1506
|
+
block_idx = block_indices[idx]
|
|
1507
|
+
actual_param = node.args.args[param_idx].arg
|
|
1508
|
+
key = (block_idx, actual_param)
|
|
1509
|
+
self._restore_alpha_mapping(key, old_mappings)
|
|
1510
|
+
|
|
1511
|
+
def _unify_joined_str(
|
|
1512
|
+
self, nodes: List[ast.JoinedStr], subst: Substitution, block_indices: List[int]
|
|
1513
|
+
) -> bool:
|
|
1514
|
+
"""
|
|
1515
|
+
Unify f-strings (JoinedStr), never parameterizing Constant children.
|
|
1516
|
+
|
|
1517
|
+
F-strings have strict structure requirements:
|
|
1518
|
+
- values list can only contain Constant or FormattedValue nodes
|
|
1519
|
+
- Constant nodes are string literals and must NEVER be parameterized
|
|
1520
|
+
- FormattedValue nodes contain expressions that CAN be unified/parameterized
|
|
1521
|
+
|
|
1522
|
+
Args:
|
|
1523
|
+
nodes: List of JoinedStr nodes
|
|
1524
|
+
subst: Current substitution
|
|
1525
|
+
block_indices: Block indices
|
|
1526
|
+
|
|
1527
|
+
Returns:
|
|
1528
|
+
True if unification succeeded
|
|
1529
|
+
"""
|
|
1530
|
+
# Check all have the same number of values
|
|
1531
|
+
values_lists = [n.values for n in nodes]
|
|
1532
|
+
if not all(len(v) == len(values_lists[0]) for v in values_lists):
|
|
1533
|
+
# Different number of components - can't unify
|
|
1534
|
+
return False
|
|
1535
|
+
|
|
1536
|
+
# Unify each component
|
|
1537
|
+
for i in range(len(values_lists[0])):
|
|
1538
|
+
components = [values[i] for values in values_lists]
|
|
1539
|
+
|
|
1540
|
+
# Check all components are the same type
|
|
1541
|
+
component_types = [type(c) for c in components]
|
|
1542
|
+
if len(set(component_types)) != 1:
|
|
1543
|
+
# Different types at this position - can't unify
|
|
1544
|
+
return False
|
|
1545
|
+
|
|
1546
|
+
first_component = components[0]
|
|
1547
|
+
|
|
1548
|
+
if isinstance(first_component, ast.Constant):
|
|
1549
|
+
# String literal parts MUST be identical - NEVER parameterize
|
|
1550
|
+
const_components = cast(List[ast.Constant], components)
|
|
1551
|
+
values = [c.value for c in const_components]
|
|
1552
|
+
if not all(v == values[0] for v in values):
|
|
1553
|
+
# Different string literals - can't unify f-strings with different text
|
|
1554
|
+
return False
|
|
1555
|
+
|
|
1556
|
+
elif isinstance(first_component, ast.FormattedValue):
|
|
1557
|
+
# FormattedValue contains an expression - unify it normally
|
|
1558
|
+
# Extract the value expressions
|
|
1559
|
+
fmt_components = cast(List[ast.FormattedValue], components)
|
|
1560
|
+
value_exprs = [c.value for c in fmt_components]
|
|
1561
|
+
if not self._unify_nodes(value_exprs, subst, block_indices):
|
|
1562
|
+
return False
|
|
1563
|
+
|
|
1564
|
+
# Also check conversion and format_spec if present
|
|
1565
|
+
conversions = [c.conversion for c in fmt_components]
|
|
1566
|
+
if not all(conv == conversions[0] for conv in conversions):
|
|
1567
|
+
return False
|
|
1568
|
+
|
|
1569
|
+
# format_spec can be None or another JoinedStr
|
|
1570
|
+
format_specs = [c.format_spec for c in fmt_components]
|
|
1571
|
+
if format_specs[0] is not None:
|
|
1572
|
+
if not all(fs is not None for fs in format_specs):
|
|
1573
|
+
return False
|
|
1574
|
+
if isinstance(format_specs[0], ast.JoinedStr):
|
|
1575
|
+
joined_specs = cast(List[ast.JoinedStr], format_specs)
|
|
1576
|
+
if not self._unify_joined_str(joined_specs, subst, block_indices):
|
|
1577
|
+
return False
|
|
1578
|
+
|
|
1579
|
+
else:
|
|
1580
|
+
# Unexpected component type
|
|
1581
|
+
return False
|
|
1582
|
+
|
|
1583
|
+
return True
|
|
1584
|
+
|
|
1585
|
+
def _unify_lists(
|
|
1586
|
+
self, lists: Sequence[Sequence[Any]], subst: Substitution, block_indices: Sequence[int]
|
|
1587
|
+
) -> bool:
|
|
1588
|
+
"""
|
|
1589
|
+
Unify lists of values.
|
|
1590
|
+
|
|
1591
|
+
Args:
|
|
1592
|
+
lists: List of lists to unify
|
|
1593
|
+
subst: Current substitution
|
|
1594
|
+
block_indices: Block indices
|
|
1595
|
+
|
|
1596
|
+
Returns:
|
|
1597
|
+
True if unification succeeded
|
|
1598
|
+
"""
|
|
1599
|
+
# All lists must have same length
|
|
1600
|
+
if not all(len(lst) == len(lists[0]) for lst in lists):
|
|
1601
|
+
return False
|
|
1602
|
+
|
|
1603
|
+
# Unify element by element
|
|
1604
|
+
for i in range(len(lists[0])):
|
|
1605
|
+
elements = [lst[i] for lst in lists]
|
|
1606
|
+
|
|
1607
|
+
# Check element types
|
|
1608
|
+
elem_types = set(type(e) for e in elements)
|
|
1609
|
+
if len(elem_types) > 1:
|
|
1610
|
+
return False
|
|
1611
|
+
|
|
1612
|
+
first_elem = elements[0]
|
|
1613
|
+
|
|
1614
|
+
if isinstance(first_elem, ast.AST):
|
|
1615
|
+
# AST nodes - unify recursively
|
|
1616
|
+
if not self._unify_nodes(elements, subst, block_indices):
|
|
1617
|
+
return False
|
|
1618
|
+
elif isinstance(first_elem, list):
|
|
1619
|
+
# Nested lists
|
|
1620
|
+
if not self._unify_lists(elements, subst, block_indices):
|
|
1621
|
+
return False
|
|
1622
|
+
else:
|
|
1623
|
+
# Primitive values - must be equal
|
|
1624
|
+
if not all(e == first_elem for e in elements):
|
|
1625
|
+
return False
|
|
1626
|
+
|
|
1627
|
+
return True
|
|
1628
|
+
|
|
1629
|
+
def _check_constant_consistency(self, values: List[Any], block_indices: Sequence[int]) -> bool:
|
|
1630
|
+
"""
|
|
1631
|
+
Check if constants can be consistently parameterized per the user's rule.
|
|
1632
|
+
|
|
1633
|
+
The rule: For constants to be parameterized, ALL occurrences across blocks
|
|
1634
|
+
must unify consistently. If a value appears at multiple positions, those
|
|
1635
|
+
positions must align and unify the same way.
|
|
1636
|
+
|
|
1637
|
+
Example that should FAIL:
|
|
1638
|
+
- Block 0: "x = item * 2" and "z = y ** 2"
|
|
1639
|
+
- Block 1: "x = item * 3" and "z = y ** 2"
|
|
1640
|
+
- Value 2 appears at 2 positions in block 0
|
|
1641
|
+
- At position 1: 2 differs from 3 (would parameterize)
|
|
1642
|
+
- At position 2: 2 equals 2 (would NOT parameterize)
|
|
1643
|
+
- INCONSISTENT -> return False
|
|
1644
|
+
|
|
1645
|
+
Args:
|
|
1646
|
+
values: List of constant values (one per block)
|
|
1647
|
+
block_indices: Block indices
|
|
1648
|
+
|
|
1649
|
+
Returns:
|
|
1650
|
+
True if constants can be consistently parameterized
|
|
1651
|
+
"""
|
|
1652
|
+
if len(values) != 2 or len(block_indices) != 2:
|
|
1653
|
+
# Only handle 2-block case for now
|
|
1654
|
+
return True
|
|
1655
|
+
|
|
1656
|
+
value0, value1 = values
|
|
1657
|
+
idx0, idx1 = block_indices
|
|
1658
|
+
|
|
1659
|
+
# Get all positions where each value appears
|
|
1660
|
+
positions0 = self.constant_positions.get((idx0, value0), [])
|
|
1661
|
+
positions1 = self.constant_positions.get((idx1, value1), [])
|
|
1662
|
+
|
|
1663
|
+
# If either value appears only once, it's trivially consistent
|
|
1664
|
+
if len(positions0) <= 1 and len(positions1) <= 1:
|
|
1665
|
+
return True
|
|
1666
|
+
|
|
1667
|
+
# If both values are the same, check they appear at same positions
|
|
1668
|
+
if value0 == value1:
|
|
1669
|
+
# Same value in both blocks - they should appear at same positions
|
|
1670
|
+
# If they do, unification will succeed without parameterization
|
|
1671
|
+
# This is fine, return True
|
|
1672
|
+
return True
|
|
1673
|
+
|
|
1674
|
+
# Different values - check consistency
|
|
1675
|
+
# If value0 appears N times, and value1 appears M times, and N != M,
|
|
1676
|
+
# this is already inconsistent
|
|
1677
|
+
if len(positions0) != len(positions1):
|
|
1678
|
+
# Different number of occurrences
|
|
1679
|
+
# This means one value appears more times than the other
|
|
1680
|
+
# Cannot parameterize consistently
|
|
1681
|
+
return False
|
|
1682
|
+
|
|
1683
|
+
# Both values appear the same number of times
|
|
1684
|
+
# Check if they appear at structurally matching positions
|
|
1685
|
+
# If the positions don't align, we can't parameterize
|
|
1686
|
+
|
|
1687
|
+
# For now, use a simple heuristic: if a value appears multiple times (> 1),
|
|
1688
|
+
# we need the positions to match exactly
|
|
1689
|
+
if len(positions0) > 1:
|
|
1690
|
+
# Sort positions for comparison
|
|
1691
|
+
sorted_pos0 = sorted(positions0)
|
|
1692
|
+
sorted_pos1 = sorted(positions1)
|
|
1693
|
+
|
|
1694
|
+
# Check if positions align
|
|
1695
|
+
if sorted_pos0 != sorted_pos1:
|
|
1696
|
+
# Positions don't align - inconsistent
|
|
1697
|
+
return False
|
|
1698
|
+
|
|
1699
|
+
# Positions align - can parameterize consistently
|
|
1700
|
+
return True
|
|
1701
|
+
|
|
1702
|
+
def _collect_constant_positions(self, blocks: List[List[ast.AST]]) -> None:
|
|
1703
|
+
"""
|
|
1704
|
+
Collect all constant occurrences and their structural positions.
|
|
1705
|
+
|
|
1706
|
+
This pre-pass finds every constant in each block and records its position
|
|
1707
|
+
using a path tuple that uniquely identifies its location in the AST.
|
|
1708
|
+
|
|
1709
|
+
Args:
|
|
1710
|
+
blocks: List of code blocks to analyze
|
|
1711
|
+
"""
|
|
1712
|
+
self.constant_positions = {}
|
|
1713
|
+
|
|
1714
|
+
for block_idx, block in enumerate(blocks):
|
|
1715
|
+
for stmt_idx, stmt in enumerate(block):
|
|
1716
|
+
# Traverse this statement and record all constants
|
|
1717
|
+
self._record_constants_in_tree(stmt, (stmt_idx,), block_idx)
|
|
1718
|
+
|
|
1719
|
+
def _record_constants_in_tree(
|
|
1720
|
+
self, node: ast.AST, path: Tuple[Any, ...], block_idx: int
|
|
1721
|
+
) -> None:
|
|
1722
|
+
"""
|
|
1723
|
+
Recursively traverse AST and record all constant positions.
|
|
1724
|
+
|
|
1725
|
+
Args:
|
|
1726
|
+
node: Current AST node
|
|
1727
|
+
path: Tuple representing path from statement root
|
|
1728
|
+
block_idx: Which block this is from
|
|
1729
|
+
"""
|
|
1730
|
+
if isinstance(node, ast.Constant):
|
|
1731
|
+
# Record this constant's position
|
|
1732
|
+
key = (block_idx, node.value)
|
|
1733
|
+
if key not in self.constant_positions:
|
|
1734
|
+
self.constant_positions[key] = []
|
|
1735
|
+
self.constant_positions[key].append(path)
|
|
1736
|
+
|
|
1737
|
+
# Recursively visit children
|
|
1738
|
+
for field_name, field_value in self._iter_child_fields(node):
|
|
1739
|
+
if isinstance(field_value, list):
|
|
1740
|
+
for i, item in enumerate(field_value):
|
|
1741
|
+
if isinstance(item, ast.AST):
|
|
1742
|
+
child_path = path + (field_name, i)
|
|
1743
|
+
self._record_constants_in_tree(item, child_path, block_idx)
|
|
1744
|
+
elif isinstance(field_value, ast.AST):
|
|
1745
|
+
child_path = path + (field_name,)
|
|
1746
|
+
self._record_constants_in_tree(field_value, child_path, block_idx)
|
|
1747
|
+
|
|
1748
|
+
@staticmethod
|
|
1749
|
+
def _iter_child_fields(node: ast.AST) -> Iterator[Tuple[str, Any]]:
|
|
1750
|
+
"""Yield (field_name, value) pairs skipping location metadata fields."""
|
|
1751
|
+
for field_name in getattr(node, "_fields", ()): # pragma: no branch - simple iteration
|
|
1752
|
+
if field_name in ("lineno", "col_offset", "end_lineno", "end_col_offset"):
|
|
1753
|
+
continue
|
|
1754
|
+
yield field_name, getattr(node, field_name, None)
|
|
1755
|
+
|
|
1756
|
+
def _find_all_occurrences(self, value: Any, block: List[ast.AST]) -> List[ast.AST]:
|
|
1757
|
+
"""
|
|
1758
|
+
Find all AST nodes in a block that are constants with the given value.
|
|
1759
|
+
|
|
1760
|
+
Args:
|
|
1761
|
+
value: The constant value to search for
|
|
1762
|
+
block: List of AST statements
|
|
1763
|
+
|
|
1764
|
+
Returns:
|
|
1765
|
+
List of ast.Constant nodes with matching value
|
|
1766
|
+
"""
|
|
1767
|
+
occurrences: List[ast.AST] = []
|
|
1768
|
+
|
|
1769
|
+
class OccurrenceFinder(ast.NodeVisitor):
|
|
1770
|
+
def visit_Constant(self, node: ast.Constant) -> None:
|
|
1771
|
+
if node.value == value:
|
|
1772
|
+
occurrences.append(node)
|
|
1773
|
+
self.generic_visit(node)
|
|
1774
|
+
|
|
1775
|
+
for stmt in block:
|
|
1776
|
+
finder = OccurrenceFinder()
|
|
1777
|
+
finder.visit(stmt)
|
|
1778
|
+
|
|
1779
|
+
return occurrences
|
|
1780
|
+
|
|
1781
|
+
def _constant_appears_identically_elsewhere(
|
|
1782
|
+
self, values: List[Any], block_indices: List[int]
|
|
1783
|
+
) -> bool:
|
|
1784
|
+
"""
|
|
1785
|
+
Check if any of the differing constant values also appears identically
|
|
1786
|
+
in both blocks at other positions.
|
|
1787
|
+
|
|
1788
|
+
This implements the consistency rule: if a constant value appears at
|
|
1789
|
+
multiple positions, it must differ consistently at all positions or
|
|
1790
|
+
be identical at all positions. Mixed behavior means we can't parameterize
|
|
1791
|
+
just the constant - we'd need to parameterize a larger expression.
|
|
1792
|
+
|
|
1793
|
+
Args:
|
|
1794
|
+
values: List of constant values (one per block)
|
|
1795
|
+
block_indices: Block indices
|
|
1796
|
+
|
|
1797
|
+
Returns:
|
|
1798
|
+
True if any value appears identically elsewhere in both blocks
|
|
1799
|
+
"""
|
|
1800
|
+
if not self.current_blocks or len(self.current_blocks) < 2:
|
|
1801
|
+
return False
|
|
1802
|
+
|
|
1803
|
+
# Check each unique value
|
|
1804
|
+
unique_values = set(values)
|
|
1805
|
+
|
|
1806
|
+
for value in unique_values:
|
|
1807
|
+
# Find all occurrences of this value in each block
|
|
1808
|
+
occurrences_per_block = []
|
|
1809
|
+
for idx in block_indices:
|
|
1810
|
+
if idx < len(self.current_blocks):
|
|
1811
|
+
occs = self._find_all_occurrences(value, self.current_blocks[idx])
|
|
1812
|
+
occurrences_per_block.append(len(occs))
|
|
1813
|
+
else:
|
|
1814
|
+
occurrences_per_block.append(0)
|
|
1815
|
+
|
|
1816
|
+
# If this value appears multiple times in ANY block, check consistency
|
|
1817
|
+
if any(count > 1 for count in occurrences_per_block):
|
|
1818
|
+
# Value appears multiple times - need to check if it's used consistently
|
|
1819
|
+
# For now, we use a conservative approach: if a value appears multiple
|
|
1820
|
+
# times, we don't parameterize the bare constant
|
|
1821
|
+
# This prevents the case where "2" appears as both "item * 2" (differs)
|
|
1822
|
+
# and "y ** 2" (same in both)
|
|
1823
|
+
return True
|
|
1824
|
+
|
|
1825
|
+
return False
|
|
1826
|
+
|
|
1827
|
+
def _try_parameterize(
|
|
1828
|
+
self, exprs: Sequence[ast.AST], subst: Substitution, block_indices: Sequence[int]
|
|
1829
|
+
) -> bool:
|
|
1830
|
+
"""
|
|
1831
|
+
Try to parameterize differing expressions.
|
|
1832
|
+
|
|
1833
|
+
Args:
|
|
1834
|
+
exprs: List of expressions that differ
|
|
1835
|
+
subst: Current substitution
|
|
1836
|
+
block_indices: Block index for each expression
|
|
1837
|
+
|
|
1838
|
+
Returns:
|
|
1839
|
+
True if parameterization succeeded
|
|
1840
|
+
"""
|
|
1841
|
+
# F-strings (JoinedStr) must not be parameterized as a whole; only their
|
|
1842
|
+
# internal expressions (FormattedValue.value) are eligible. Prevent turning
|
|
1843
|
+
# entire f-strings into a single parameter to preserve structure.
|
|
1844
|
+
if any(isinstance(expr, ast.JoinedStr) for expr in exprs):
|
|
1845
|
+
return False
|
|
1846
|
+
|
|
1847
|
+
# CRITICAL: Cannot parameterize statement nodes (only expression nodes)
|
|
1848
|
+
# Statements (If, For, FunctionDef, etc.) must have the same type to unify
|
|
1849
|
+
# Only expressions (Name, Constant, Call, etc.) can be parameterized
|
|
1850
|
+
if any(isinstance(expr, ast.stmt) for expr in exprs):
|
|
1851
|
+
return False
|
|
1852
|
+
|
|
1853
|
+
# Check if we've already parameterized these exact expressions
|
|
1854
|
+
|
|
1855
|
+
# Check if all expressions are already mapped to the same parameter
|
|
1856
|
+
existing_params = [
|
|
1857
|
+
subst.get_param_for_expr(idx, expr) for idx, expr in zip(block_indices, exprs)
|
|
1858
|
+
]
|
|
1859
|
+
|
|
1860
|
+
if all(p is not None for p in existing_params):
|
|
1861
|
+
# All mapped - check they map to the same parameter
|
|
1862
|
+
if len(set(existing_params)) == 1:
|
|
1863
|
+
return True
|
|
1864
|
+
else:
|
|
1865
|
+
# Mapped to different parameters - can't unify
|
|
1866
|
+
return False
|
|
1867
|
+
|
|
1868
|
+
# Check we haven't exceeded max parameters
|
|
1869
|
+
if len(subst.param_expressions) >= self.max_parameters:
|
|
1870
|
+
return False
|
|
1871
|
+
|
|
1872
|
+
# Analyze bound variables in each expression
|
|
1873
|
+
# For each expression, find which variables it references that are bound in context
|
|
1874
|
+
bound_vars_per_expr = []
|
|
1875
|
+
for idx, expr in zip(block_indices, exprs):
|
|
1876
|
+
if self.current_blocks and idx < len(self.current_blocks):
|
|
1877
|
+
# Get the full block as context
|
|
1878
|
+
block = self.current_blocks[idx]
|
|
1879
|
+
# Find which variables in the expression are bound in the block context
|
|
1880
|
+
# mypy: ast.Module expects list[ast.stmt]
|
|
1881
|
+
typed_block = cast(List[ast.stmt], block)
|
|
1882
|
+
bound_in_context = get_bound_variables_in_context(
|
|
1883
|
+
ast.Module(body=typed_block, type_ignores=[]), expr
|
|
1884
|
+
)
|
|
1885
|
+
# Get variables referenced in the expression
|
|
1886
|
+
vars_in_expr = get_free_variables(expr)
|
|
1887
|
+
# Intersection: bound variables that are actually used in expression
|
|
1888
|
+
bound_vars = bound_in_context & vars_in_expr
|
|
1889
|
+
|
|
1890
|
+
bound_vars_per_expr.append(bound_vars)
|
|
1891
|
+
else:
|
|
1892
|
+
bound_vars_per_expr.append(set())
|
|
1893
|
+
|
|
1894
|
+
# Check if all expressions reference the same bound variables
|
|
1895
|
+
# If they do, this should be a function parameter
|
|
1896
|
+
all_bound_vars = [sorted(bv) for bv in bound_vars_per_expr]
|
|
1897
|
+
|
|
1898
|
+
# Determine common bound variables (should be same across all expressions)
|
|
1899
|
+
if all_bound_vars and len(set(tuple(bv) for bv in all_bound_vars)) == 1:
|
|
1900
|
+
# All expressions reference the same set of bound variables
|
|
1901
|
+
common_bound_vars = all_bound_vars[0] if all_bound_vars[0] else None
|
|
1902
|
+
else:
|
|
1903
|
+
# Different bound variables - use None (not a function parameter)
|
|
1904
|
+
common_bound_vars = None
|
|
1905
|
+
|
|
1906
|
+
# CRITICAL: Check if bound variables are accessible at call site
|
|
1907
|
+
# Comprehension variables (for r in results) are NOT accessible at call site
|
|
1908
|
+
# Only function-level free variables can be lambda-lifted
|
|
1909
|
+
if common_bound_vars:
|
|
1910
|
+
# Check if these bound variables are accessible at the function call site
|
|
1911
|
+
# (i.e., they're free variables of the block, not just comprehension variables)
|
|
1912
|
+
for idx, expr in zip(block_indices, exprs):
|
|
1913
|
+
if self.current_blocks is not None and idx < len(self.current_blocks):
|
|
1914
|
+
from .scope_analyzer import ScopeAnalyzer
|
|
1915
|
+
from typing import Any as _Any, cast as _cast
|
|
1916
|
+
|
|
1917
|
+
analyzer1 = _cast(_Any, ScopeAnalyzer)()
|
|
1918
|
+
block_free_vars = analyzer1.get_free_variables(self.current_blocks[idx])
|
|
1919
|
+
|
|
1920
|
+
# Check if all bound variables used in the expression are free variables
|
|
1921
|
+
for var in common_bound_vars:
|
|
1922
|
+
if var not in block_free_vars:
|
|
1923
|
+
# Bound variable (like comprehension var) not accessible at call site
|
|
1924
|
+
# Cannot lambda-lift - refuse to parameterize
|
|
1925
|
+
return False
|
|
1926
|
+
|
|
1927
|
+
# CRITICAL: If expressions are simple names without bound variables,
|
|
1928
|
+
# they must be validated - they need to exist at the call site
|
|
1929
|
+
# (Unless they're being lambda-lifted, in which case common_bound_vars is not None)
|
|
1930
|
+
if common_bound_vars is None:
|
|
1931
|
+
# Not using lambda lifting - check if expressions reference undefined variables
|
|
1932
|
+
for idx, expr in zip(block_indices, exprs):
|
|
1933
|
+
if isinstance(expr, ast.Name):
|
|
1934
|
+
# Simple variable reference - needs to exist at call site
|
|
1935
|
+
# Get free variables of the entire block to see what's available
|
|
1936
|
+
if self.current_blocks is not None and idx < len(self.current_blocks):
|
|
1937
|
+
from .scope_analyzer import ScopeAnalyzer
|
|
1938
|
+
from typing import Any as _Any, cast as _cast
|
|
1939
|
+
|
|
1940
|
+
analyzer2 = _cast(_Any, ScopeAnalyzer)()
|
|
1941
|
+
block_free_vars = analyzer2.get_free_variables(self.current_blocks[idx])
|
|
1942
|
+
|
|
1943
|
+
# Check if this variable is available at call site
|
|
1944
|
+
if expr.id not in block_free_vars:
|
|
1945
|
+
# Variable not available at call site - can't parameterize
|
|
1946
|
+
import logging
|
|
1947
|
+
|
|
1948
|
+
logger = logging.getLogger(__name__)
|
|
1949
|
+
logger.debug(
|
|
1950
|
+
f"Skipping refactoring: Variable '{expr.id}' is not accessible at function scope. "
|
|
1951
|
+
f"It may be defined inside a nested function or be an unbound variable reference."
|
|
1952
|
+
)
|
|
1953
|
+
return False
|
|
1954
|
+
|
|
1955
|
+
# Create a new parameter
|
|
1956
|
+
# Use __ prefix to avoid name collisions (Python convention)
|
|
1957
|
+
param_name = f"__param_{self.param_counter}"
|
|
1958
|
+
self.param_counter += 1
|
|
1959
|
+
|
|
1960
|
+
# Add mappings for each block
|
|
1961
|
+
for idx, expr in zip(block_indices, exprs):
|
|
1962
|
+
subst.add_mapping(idx, expr, param_name, bound_vars=common_bound_vars)
|
|
1963
|
+
|
|
1964
|
+
return True
|
|
1965
|
+
|
|
1966
|
+
def _setup_bound_variable_alpha_renamings(self, blocks: List[List[ast.AST]]) -> None:
|
|
1967
|
+
"""
|
|
1968
|
+
Setup alpha-renamings for block-level bound variables.
|
|
1969
|
+
|
|
1970
|
+
This allows unification of blocks with structurally identical code but different
|
|
1971
|
+
bound variable names (e.g., 'result' vs 'output').
|
|
1972
|
+
|
|
1973
|
+
Strategy:
|
|
1974
|
+
1. For each block, collect variables that are first bound (assigned) in that block
|
|
1975
|
+
2. Match variables across blocks by their structural position (where they're first assigned)
|
|
1976
|
+
3. Add alpha-renamings to map corresponding variables to canonical names
|
|
1977
|
+
|
|
1978
|
+
Example:
|
|
1979
|
+
Block1: result = x + 1; result = result + 2; return result
|
|
1980
|
+
Block2: output = y + 1; output = output + 2; return output
|
|
1981
|
+
|
|
1982
|
+
-> result and output both bound at position (0, 0)
|
|
1983
|
+
-> Add mapping: result → temp, output → temp
|
|
1984
|
+
-> Existing Name node handling will use these mappings
|
|
1985
|
+
"""
|
|
1986
|
+
# Collect binding information for each block
|
|
1987
|
+
binding_info = []
|
|
1988
|
+
for block_idx, block in enumerate(blocks):
|
|
1989
|
+
bindings = {} # var_name → (stmt_idx, target_idx)
|
|
1990
|
+
seen_vars = set()
|
|
1991
|
+
|
|
1992
|
+
for stmt_idx, stmt in enumerate(block):
|
|
1993
|
+
# Get all assignment targets in this statement
|
|
1994
|
+
targets = self._get_assignment_targets(stmt)
|
|
1995
|
+
|
|
1996
|
+
for target_idx, target in enumerate(targets):
|
|
1997
|
+
if isinstance(target, ast.Name) and isinstance(target.ctx, ast.Store):
|
|
1998
|
+
var_name = target.id
|
|
1999
|
+
# Track first binding position
|
|
2000
|
+
if var_name not in seen_vars:
|
|
2001
|
+
bindings[var_name] = (stmt_idx, target_idx)
|
|
2002
|
+
seen_vars.add(var_name)
|
|
2003
|
+
|
|
2004
|
+
binding_info.append(bindings)
|
|
2005
|
+
|
|
2006
|
+
# Match variables across blocks by structural position
|
|
2007
|
+
# position_to_vars maps (stmt_idx, target_idx) → list of (block_idx, var_name) pairs
|
|
2008
|
+
position_to_vars: Dict[Tuple[int, int], List[Tuple[int, str]]] = {}
|
|
2009
|
+
|
|
2010
|
+
for block_idx, bindings in enumerate(binding_info):
|
|
2011
|
+
for var_name, position in bindings.items():
|
|
2012
|
+
if position not in position_to_vars:
|
|
2013
|
+
position_to_vars[position] = []
|
|
2014
|
+
position_to_vars[position].append((block_idx, var_name))
|
|
2015
|
+
|
|
2016
|
+
# Generate alpha-renamings for variables at same position
|
|
2017
|
+
canonical_counter = 0
|
|
2018
|
+
used_canonical_names = set()
|
|
2019
|
+
|
|
2020
|
+
for position, var_list in position_to_vars.items():
|
|
2021
|
+
# Only create alpha-renaming if multiple blocks have a variable at this position
|
|
2022
|
+
if len(var_list) < 2:
|
|
2023
|
+
continue
|
|
2024
|
+
|
|
2025
|
+
# Check if variables at this position have different names
|
|
2026
|
+
var_names = [var_name for _, var_name in var_list]
|
|
2027
|
+
if len(set(var_names)) <= 1:
|
|
2028
|
+
# All same name - no renaming needed
|
|
2029
|
+
continue
|
|
2030
|
+
|
|
2031
|
+
# Generate canonical name
|
|
2032
|
+
canonical_name = f"__temp_{canonical_counter}"
|
|
2033
|
+
while canonical_name in used_canonical_names:
|
|
2034
|
+
canonical_counter += 1
|
|
2035
|
+
canonical_name = f"__temp_{canonical_counter}"
|
|
2036
|
+
used_canonical_names.add(canonical_name)
|
|
2037
|
+
canonical_counter += 1
|
|
2038
|
+
|
|
2039
|
+
# Add alpha-renamings for all blocks
|
|
2040
|
+
for block_idx, var_name in var_list:
|
|
2041
|
+
key = (block_idx, var_name)
|
|
2042
|
+
self.alpha_renamings[key] = canonical_name
|
|
2043
|
+
|
|
2044
|
+
def _get_assignment_targets(self, stmt: ast.AST) -> List[ast.AST]:
|
|
2045
|
+
"""
|
|
2046
|
+
Extract assignment targets from a statement.
|
|
2047
|
+
|
|
2048
|
+
Returns list of target AST nodes (may be Name, Tuple, List, etc.)
|
|
2049
|
+
"""
|
|
2050
|
+
targets = []
|
|
2051
|
+
|
|
2052
|
+
if isinstance(stmt, ast.Assign):
|
|
2053
|
+
# Regular assignment: x = 1 or x, y = 1, 2
|
|
2054
|
+
for target in stmt.targets:
|
|
2055
|
+
targets.extend(self._flatten_assignment_target(target))
|
|
2056
|
+
elif isinstance(stmt, ast.AugAssign):
|
|
2057
|
+
# Augmented assignment: x += 1
|
|
2058
|
+
targets.append(stmt.target)
|
|
2059
|
+
elif isinstance(stmt, ast.AnnAssign):
|
|
2060
|
+
# Annotated assignment: x: int = 1
|
|
2061
|
+
if stmt.target:
|
|
2062
|
+
targets.append(stmt.target)
|
|
2063
|
+
elif isinstance(stmt, ast.With):
|
|
2064
|
+
# With bindings: with expr as target
|
|
2065
|
+
for item in stmt.items:
|
|
2066
|
+
if item.optional_vars is not None:
|
|
2067
|
+
targets.extend(self._flatten_assignment_target(item.optional_vars))
|
|
2068
|
+
elif isinstance(stmt, ast.Try):
|
|
2069
|
+
# Except handler names are bindings
|
|
2070
|
+
for handler in stmt.handlers:
|
|
2071
|
+
if handler.name:
|
|
2072
|
+
targets.append(ast.Name(id=handler.name, ctx=ast.Store()))
|
|
2073
|
+
|
|
2074
|
+
return targets
|
|
2075
|
+
|
|
2076
|
+
def _flatten_assignment_target(self, target: ast.AST) -> List[ast.AST]:
|
|
2077
|
+
"""
|
|
2078
|
+
Flatten assignment target to individual names.
|
|
2079
|
+
|
|
2080
|
+
Examples:
|
|
2081
|
+
x → [x]
|
|
2082
|
+
(x, y) → [x, y]
|
|
2083
|
+
[x, (y, z)] → [x, y, z]
|
|
2084
|
+
"""
|
|
2085
|
+
if isinstance(target, ast.Name):
|
|
2086
|
+
return [target]
|
|
2087
|
+
elif isinstance(target, (ast.Tuple, ast.List)):
|
|
2088
|
+
result = []
|
|
2089
|
+
for elt in target.elts:
|
|
2090
|
+
result.extend(self._flatten_assignment_target(elt))
|
|
2091
|
+
return result
|
|
2092
|
+
else:
|
|
2093
|
+
# Other complex targets (subscript, attribute, etc.) - don't extract names
|
|
2094
|
+
return []
|
|
2095
|
+
|
|
2096
|
+
def reset(self) -> None:
|
|
2097
|
+
"""Reset the parameter counter."""
|
|
2098
|
+
self.param_counter = 0
|
|
2099
|
+
|
|
2100
|
+
def _set_feature_flags(
|
|
2101
|
+
self, parameterize_constants: bool, promote_equal_hof_literals: bool
|
|
2102
|
+
) -> None:
|
|
2103
|
+
self.parameterize_constants = parameterize_constants
|
|
2104
|
+
self.promote_equal_hof_literals = promote_equal_hof_literals
|
|
2105
|
+
|
|
2106
|
+
def _reset_unification_state(self, blocks: List[List[ast.AST]]) -> None:
|
|
2107
|
+
self.param_counter = 0
|
|
2108
|
+
self.current_blocks = blocks
|
|
2109
|
+
|
|
2110
|
+
def _assign_alpha_mapping(
|
|
2111
|
+
self,
|
|
2112
|
+
key: Tuple[int, str],
|
|
2113
|
+
canonical: str,
|
|
2114
|
+
old_mappings: Dict[Tuple[int, str], str],
|
|
2115
|
+
) -> None:
|
|
2116
|
+
if key in self.alpha_renamings:
|
|
2117
|
+
old_mappings[key] = self.alpha_renamings[key]
|
|
2118
|
+
self.alpha_renamings[key] = canonical
|
|
2119
|
+
|
|
2120
|
+
def _restore_alpha_mapping(
|
|
2121
|
+
self,
|
|
2122
|
+
key: Tuple[int, str],
|
|
2123
|
+
old_mappings: Dict[Tuple[int, str], str],
|
|
2124
|
+
) -> None:
|
|
2125
|
+
if key in old_mappings:
|
|
2126
|
+
self.alpha_renamings[key] = old_mappings[key]
|
|
2127
|
+
else:
|
|
2128
|
+
self.alpha_renamings.pop(key, None)
|