code-towel 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- code_towel-1.0.0.dist-info/METADATA +722 -0
- code_towel-1.0.0.dist-info/RECORD +27 -0
- code_towel-1.0.0.dist-info/WHEEL +5 -0
- code_towel-1.0.0.dist-info/entry_points.txt +4 -0
- code_towel-1.0.0.dist-info/licenses/LICENSE +201 -0
- code_towel-1.0.0.dist-info/top_level.txt +1 -0
- towel/__init__.py +20 -0
- towel/cli.py +813 -0
- towel/unification/__init__.py +24 -0
- towel/unification/assignment_analyzer.py +403 -0
- towel/unification/ast_normalizer.py +223 -0
- towel/unification/ast_pretty_printer.py +133 -0
- towel/unification/binding_detector.py +381 -0
- towel/unification/block_signature.py +123 -0
- towel/unification/builtins.py +209 -0
- towel/unification/exceptions.py +113 -0
- towel/unification/extractor.py +884 -0
- towel/unification/models.py +190 -0
- towel/unification/nominal_unifier.py +375 -0
- towel/unification/orphan_detector.py +187 -0
- towel/unification/pipeline.py +524 -0
- towel/unification/project_layout.py +201 -0
- towel/unification/refactor_engine.py +3381 -0
- towel/unification/scope_analyzer.py +597 -0
- towel/unification/unifier.py +2128 -0
- towel/unification/visitor_utils.py +83 -0
- towel/unification/visitors.py +377 -0
|
@@ -0,0 +1,3381 @@
|
|
|
1
|
+
# Copyright 2025 Eric Allen
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
|
|
15
|
+
"""
|
|
16
|
+
Main refactoring engine using unification.
|
|
17
|
+
|
|
18
|
+
This orchestrates the entire refactoring process:
|
|
19
|
+
1. Parse files into ASTs
|
|
20
|
+
2. Find pairs of code blocks in top-level functions
|
|
21
|
+
3. Attempt unification to find parameterizable differences
|
|
22
|
+
4. Extract functions hygienically if unification succeeds
|
|
23
|
+
5. Generate replacement calls
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
import ast
|
|
27
|
+
import copy
|
|
28
|
+
import os
|
|
29
|
+
import re
|
|
30
|
+
from collections import deque
|
|
31
|
+
from concurrent.futures import ProcessPoolExecutor, as_completed
|
|
32
|
+
from dataclasses import dataclass
|
|
33
|
+
from typing import Any, List, Tuple, Dict, Set, Optional, FrozenSet, Literal, Union, Sequence, cast
|
|
34
|
+
from weakref import WeakKeyDictionary
|
|
35
|
+
from pathlib import Path
|
|
36
|
+
from .scope_analyzer import ScopeAnalyzer, Scope
|
|
37
|
+
from .unifier import Unifier
|
|
38
|
+
from .extractor import HygienicExtractor, is_value_producing
|
|
39
|
+
from .orphan_detector import has_orphaned_variables
|
|
40
|
+
from .assignment_analyzer import (
|
|
41
|
+
analyze_assignments,
|
|
42
|
+
has_reassignments_without_bindings,
|
|
43
|
+
_collect_block_binding_stats,
|
|
44
|
+
_collect_bindings_and_reassignments,
|
|
45
|
+
)
|
|
46
|
+
from .project_layout import ProjectLayout
|
|
47
|
+
from .block_signature import extract_block_signature, quick_filter
|
|
48
|
+
from .models import (
|
|
49
|
+
CodeBlockPair,
|
|
50
|
+
MethodInfo,
|
|
51
|
+
ClassInfo,
|
|
52
|
+
ClassInsertionPlan,
|
|
53
|
+
Replacement,
|
|
54
|
+
RefactoringProposal,
|
|
55
|
+
)
|
|
56
|
+
from .pipeline import run_pipeline
|
|
57
|
+
from .visitors import (
|
|
58
|
+
MethodCallRewriter,
|
|
59
|
+
LoopReturnFinder,
|
|
60
|
+
NameCollector,
|
|
61
|
+
AugAssignFinder,
|
|
62
|
+
AssignTargetVisitor,
|
|
63
|
+
ClassLocator,
|
|
64
|
+
FuncLocator,
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
# Configuration defaults
|
|
68
|
+
DEFAULT_MAX_PARAMETERS = 5
|
|
69
|
+
DEFAULT_MIN_LINES = 4
|
|
70
|
+
DEFAULT_SIMILARITY_THRESHOLD = 0.6
|
|
71
|
+
DEFAULT_MAX_ITERATIONS = 0 # Unlimited
|
|
72
|
+
|
|
73
|
+
FunctionNode = Union[ast.FunctionDef, ast.AsyncFunctionDef]
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
@dataclass(frozen=True)
|
|
77
|
+
class BlockBindingSnapshot:
|
|
78
|
+
"""Summarized binding data for a block, mirroring DRY helper structure."""
|
|
79
|
+
|
|
80
|
+
bound_in_block: Set[str]
|
|
81
|
+
reassigned_in_block: Set[str]
|
|
82
|
+
bound_before_block: Set[str]
|
|
83
|
+
bound_after_block: Set[str]
|
|
84
|
+
initially_bound: Set[str]
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
_worker_engine: Optional["UnificationRefactorEngine"] = None
|
|
88
|
+
_worker_functions: Optional[
|
|
89
|
+
List[
|
|
90
|
+
Tuple[
|
|
91
|
+
str,
|
|
92
|
+
FunctionNode,
|
|
93
|
+
str,
|
|
94
|
+
"ScopeAnalyzer",
|
|
95
|
+
"Scope",
|
|
96
|
+
Optional[str],
|
|
97
|
+
Optional[str],
|
|
98
|
+
List[str],
|
|
99
|
+
]
|
|
100
|
+
]
|
|
101
|
+
] = None
|
|
102
|
+
_worker_class_infos: Optional[List[ClassInfo]] = None
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def _initialize_pair_worker(
|
|
106
|
+
engine_config: Dict[str, Any],
|
|
107
|
+
functions: List[
|
|
108
|
+
Tuple[str, FunctionNode, str, ScopeAnalyzer, Scope, Optional[str], Optional[str], List[str]]
|
|
109
|
+
],
|
|
110
|
+
class_infos: List[ClassInfo],
|
|
111
|
+
) -> None:
|
|
112
|
+
"""Initializer that primes each worker process with engine state and function context."""
|
|
113
|
+
|
|
114
|
+
global _worker_engine, _worker_functions, _worker_class_infos
|
|
115
|
+
|
|
116
|
+
max_params = engine_config["max_parameters"]
|
|
117
|
+
min_ln = engine_config["min_lines"]
|
|
118
|
+
_worker_engine = UnificationRefactorEngine(
|
|
119
|
+
max_parameters=int(max_params) if max_params is not None else DEFAULT_MAX_PARAMETERS,
|
|
120
|
+
min_lines=int(min_ln) if min_ln is not None else DEFAULT_MIN_LINES,
|
|
121
|
+
parameterize_constants=bool(engine_config["parameterize_constants"]),
|
|
122
|
+
prefer_absolute_imports=engine_config.get("prefer_absolute_imports"),
|
|
123
|
+
pep420_namespace_packages=engine_config.get("pep420_namespace_packages"),
|
|
124
|
+
)
|
|
125
|
+
_worker_functions = functions
|
|
126
|
+
_worker_class_infos = class_infos
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _process_pair_in_worker(
|
|
130
|
+
task: Tuple[int, CodeBlockPair],
|
|
131
|
+
) -> Tuple[int, Optional[RefactoringProposal]]:
|
|
132
|
+
"""Worker entry point that evaluates a single code block pair."""
|
|
133
|
+
|
|
134
|
+
if _worker_engine is None or _worker_functions is None or _worker_class_infos is None:
|
|
135
|
+
raise RuntimeError("Worker not initialized for pair processing")
|
|
136
|
+
|
|
137
|
+
pair_index, pair = task
|
|
138
|
+
proposal = _worker_engine._try_refactor_pair_multi_file(
|
|
139
|
+
pair, _worker_functions, _worker_class_infos
|
|
140
|
+
)
|
|
141
|
+
return pair_index, proposal
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
class UnificationRefactorEngine:
|
|
145
|
+
"""
|
|
146
|
+
Main engine for unification-based refactoring.
|
|
147
|
+
|
|
148
|
+
This finds and extracts duplicate code using unification.
|
|
149
|
+
"""
|
|
150
|
+
|
|
151
|
+
def __init__(
|
|
152
|
+
self,
|
|
153
|
+
max_parameters: int = DEFAULT_MAX_PARAMETERS,
|
|
154
|
+
min_lines: int = DEFAULT_MIN_LINES,
|
|
155
|
+
parameterize_constants: bool = True,
|
|
156
|
+
*,
|
|
157
|
+
prefer_absolute_imports: Optional[bool] = None,
|
|
158
|
+
pep420_namespace_packages: Optional[bool] = None,
|
|
159
|
+
promote_equal_hof_literals: bool = False,
|
|
160
|
+
):
|
|
161
|
+
"""
|
|
162
|
+
Initialize the refactoring engine.
|
|
163
|
+
|
|
164
|
+
Args:
|
|
165
|
+
max_parameters: Maximum parameters for extracted functions (default: 5)
|
|
166
|
+
min_lines: Minimum lines for a code block (default: 4)
|
|
167
|
+
parameterize_constants: Whether to parameterize differing constants
|
|
168
|
+
"""
|
|
169
|
+
self.max_parameters = max_parameters
|
|
170
|
+
self.min_lines = min_lines
|
|
171
|
+
self.parameterize_constants = parameterize_constants
|
|
172
|
+
self.unifier = Unifier(
|
|
173
|
+
max_parameters=max_parameters,
|
|
174
|
+
parameterize_constants=parameterize_constants,
|
|
175
|
+
promote_equal_hof_literals=promote_equal_hof_literals,
|
|
176
|
+
)
|
|
177
|
+
self.extractor = HygienicExtractor()
|
|
178
|
+
# Cross-file import preferences
|
|
179
|
+
self.prefer_absolute_imports = prefer_absolute_imports
|
|
180
|
+
self.pep420_namespace_packages = pep420_namespace_packages
|
|
181
|
+
# Default behavior: allow safe handling of globals/nonlocals by not parameterizing
|
|
182
|
+
# them and promoting necessary declarations into the extracted function when needed.
|
|
183
|
+
|
|
184
|
+
# Memoization caches keyed by the identity of AST nodes parsed for this engine run.
|
|
185
|
+
self._assignment_cache: WeakKeyDictionary[ast.AST, Dict[int, bool]] = WeakKeyDictionary()
|
|
186
|
+
self._used_names_cache: WeakKeyDictionary[ast.AST, FrozenSet[str]] = WeakKeyDictionary()
|
|
187
|
+
# Track helper name allocation per canonical file so helpers remain unique.
|
|
188
|
+
self._helper_name_counters: Dict[str, int] = {}
|
|
189
|
+
|
|
190
|
+
# --- Debug helpers ---
|
|
191
|
+
def _debug_reject(
|
|
192
|
+
self, reason: str, pair: "CodeBlockPair", detail: Optional[str] = None
|
|
193
|
+
) -> None:
|
|
194
|
+
"""Emit a concise rejection line when DEBUG_PROPOSAL_REJECTIONS is set.
|
|
195
|
+
|
|
196
|
+
Includes function names and basic block ranges to help triage pruning gates.
|
|
197
|
+
"""
|
|
198
|
+
try:
|
|
199
|
+
import os as _os
|
|
200
|
+
|
|
201
|
+
if not _os.getenv("DEBUG_PROPOSAL_REJECTIONS"):
|
|
202
|
+
return
|
|
203
|
+
msg = (
|
|
204
|
+
f"REJECT[{reason}]: {pair.function1_name}{'@'+str(pair.block1_range) if pair.block1_range else ''} "
|
|
205
|
+
f"<-> {pair.function2_name}{'@'+str(pair.block2_range) if pair.block2_range else ''}"
|
|
206
|
+
)
|
|
207
|
+
if detail:
|
|
208
|
+
msg += f" :: {detail}"
|
|
209
|
+
print(msg)
|
|
210
|
+
except Exception:
|
|
211
|
+
# Never let debug logging interfere with refactoring
|
|
212
|
+
pass
|
|
213
|
+
|
|
214
|
+
def analyze_file(self, file_path: str) -> List[RefactoringProposal]:
|
|
215
|
+
"""
|
|
216
|
+
Analyze a Python file and find refactoring opportunities.
|
|
217
|
+
|
|
218
|
+
Args:
|
|
219
|
+
file_path: Path to Python file
|
|
220
|
+
|
|
221
|
+
Returns:
|
|
222
|
+
List of refactoring proposals
|
|
223
|
+
"""
|
|
224
|
+
return self.analyze_files([file_path])
|
|
225
|
+
|
|
226
|
+
def analyze_directory(
|
|
227
|
+
self,
|
|
228
|
+
directory: str,
|
|
229
|
+
recursive: bool = True,
|
|
230
|
+
*,
|
|
231
|
+
verbose: bool = False,
|
|
232
|
+
progress: str = "tqdm",
|
|
233
|
+
) -> List[RefactoringProposal]:
|
|
234
|
+
"""
|
|
235
|
+
Analyze all Python files in a directory and find refactoring opportunities.
|
|
236
|
+
|
|
237
|
+
Args:
|
|
238
|
+
directory: Path to directory
|
|
239
|
+
recursive: Whether to search subdirectories (default: True)
|
|
240
|
+
|
|
241
|
+
Returns:
|
|
242
|
+
List of refactoring proposals
|
|
243
|
+
"""
|
|
244
|
+
# Find all Python files
|
|
245
|
+
python_files = self._find_python_files(directory, recursive)
|
|
246
|
+
|
|
247
|
+
if not python_files:
|
|
248
|
+
return []
|
|
249
|
+
|
|
250
|
+
if verbose:
|
|
251
|
+
print(f"Found {len(python_files)} Python files in {directory}")
|
|
252
|
+
|
|
253
|
+
# Analyze all files together
|
|
254
|
+
return self.analyze_files(python_files, verbose=verbose, progress=progress)
|
|
255
|
+
|
|
256
|
+
def _find_python_files(self, directory: str, recursive: bool = True) -> List[str]:
|
|
257
|
+
"""
|
|
258
|
+
Find all Python files in a directory.
|
|
259
|
+
|
|
260
|
+
Args:
|
|
261
|
+
directory: Directory to search
|
|
262
|
+
recursive: Whether to search subdirectories
|
|
263
|
+
|
|
264
|
+
Returns:
|
|
265
|
+
List of Python file paths
|
|
266
|
+
"""
|
|
267
|
+
python_files = []
|
|
268
|
+
directory_path = Path(directory)
|
|
269
|
+
|
|
270
|
+
if not directory_path.exists():
|
|
271
|
+
return []
|
|
272
|
+
|
|
273
|
+
if recursive:
|
|
274
|
+
# Recursively find all .py files
|
|
275
|
+
for py_file in directory_path.rglob("*.py"):
|
|
276
|
+
# Skip common directories to ignore
|
|
277
|
+
if any(
|
|
278
|
+
part.startswith(".") or part in ["__pycache__", "venv", "env", "node_modules"]
|
|
279
|
+
for part in py_file.parts
|
|
280
|
+
):
|
|
281
|
+
continue
|
|
282
|
+
python_files.append(str(py_file))
|
|
283
|
+
else:
|
|
284
|
+
# Only find .py files in this directory
|
|
285
|
+
for py_file in directory_path.glob("*.py"):
|
|
286
|
+
python_files.append(str(py_file))
|
|
287
|
+
|
|
288
|
+
return sorted(python_files)
|
|
289
|
+
|
|
290
|
+
def _get_assignment_reuse(self, func: FunctionNode) -> Dict[int, bool]:
|
|
291
|
+
"""Return (and cache) assignment analysis for a function definition."""
|
|
292
|
+
cached = self._assignment_cache.get(func)
|
|
293
|
+
if cached is not None:
|
|
294
|
+
return cached
|
|
295
|
+
analysis = analyze_assignments(func)
|
|
296
|
+
self._assignment_cache[func] = analysis
|
|
297
|
+
return analysis
|
|
298
|
+
|
|
299
|
+
def analyze_files(
|
|
300
|
+
self,
|
|
301
|
+
file_paths: List[str],
|
|
302
|
+
*,
|
|
303
|
+
verbose: bool = False,
|
|
304
|
+
progress: str = "tqdm",
|
|
305
|
+
invalidate_paths: Optional[List[str]] = None,
|
|
306
|
+
) -> List[RefactoringProposal]:
|
|
307
|
+
"""Analyze multiple files using the compiler-style pipeline and return proposals.
|
|
308
|
+
|
|
309
|
+
invalidate_paths: If provided, forces reparse/reanalysis of these paths even if cached.
|
|
310
|
+
"""
|
|
311
|
+
# Delegate to the pipeline for analysis while preserving existing behavior
|
|
312
|
+
return run_pipeline(
|
|
313
|
+
file_paths,
|
|
314
|
+
engine=self,
|
|
315
|
+
verbose=verbose,
|
|
316
|
+
progress=progress,
|
|
317
|
+
invalidate_paths=invalidate_paths,
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
def _process_block_pairs(
|
|
321
|
+
self,
|
|
322
|
+
block_pairs: List[CodeBlockPair],
|
|
323
|
+
all_functions: List[
|
|
324
|
+
Tuple[
|
|
325
|
+
str,
|
|
326
|
+
FunctionNode,
|
|
327
|
+
str,
|
|
328
|
+
ScopeAnalyzer,
|
|
329
|
+
Scope,
|
|
330
|
+
Optional[str],
|
|
331
|
+
Optional[str],
|
|
332
|
+
List[str],
|
|
333
|
+
]
|
|
334
|
+
],
|
|
335
|
+
class_infos: List[ClassInfo],
|
|
336
|
+
*,
|
|
337
|
+
verbose: bool,
|
|
338
|
+
progress: str,
|
|
339
|
+
) -> List[RefactoringProposal]:
|
|
340
|
+
if not block_pairs:
|
|
341
|
+
return []
|
|
342
|
+
|
|
343
|
+
if self._should_use_parallel(len(block_pairs)):
|
|
344
|
+
try:
|
|
345
|
+
return self._evaluate_pairs_parallel(
|
|
346
|
+
block_pairs,
|
|
347
|
+
all_functions,
|
|
348
|
+
class_infos,
|
|
349
|
+
verbose=verbose,
|
|
350
|
+
progress=progress,
|
|
351
|
+
)
|
|
352
|
+
except Exception:
|
|
353
|
+
if verbose:
|
|
354
|
+
print("Parallel pair evaluation failed; falling back to serial execution")
|
|
355
|
+
# Fall back to serial evaluation if multiprocessing encounters an issue
|
|
356
|
+
return self._evaluate_pairs_serial(
|
|
357
|
+
block_pairs,
|
|
358
|
+
all_functions,
|
|
359
|
+
class_infos,
|
|
360
|
+
verbose=verbose,
|
|
361
|
+
progress=progress,
|
|
362
|
+
)
|
|
363
|
+
|
|
364
|
+
return self._evaluate_pairs_serial(
|
|
365
|
+
block_pairs,
|
|
366
|
+
all_functions,
|
|
367
|
+
class_infos,
|
|
368
|
+
verbose=verbose,
|
|
369
|
+
progress=progress,
|
|
370
|
+
)
|
|
371
|
+
|
|
372
|
+
def _should_use_parallel(self, pair_count: int) -> bool:
|
|
373
|
+
"""Decide if multiprocessing should be used for pair evaluation.
|
|
374
|
+
|
|
375
|
+
Benchmarks run in Nov 2025 showed the ProcessPoolExecutor path to be
|
|
376
|
+
roughly 0.6× slower than the serial evaluator because the work per
|
|
377
|
+
candidate is too small to amortize process start and AST pickling cost.
|
|
378
|
+
We keep this guard so future batching/tuning work can flip the switch
|
|
379
|
+
without reworking the call site, but for now we always stay serial.
|
|
380
|
+
"""
|
|
381
|
+
return False
|
|
382
|
+
|
|
383
|
+
@staticmethod
|
|
384
|
+
def _load_tqdm_wrapper() -> Optional[Any]:
|
|
385
|
+
"""Best-effort tqdm importer (mirrors DRY run helper, see docs/DRY_RUN_2025-11-28.md)."""
|
|
386
|
+
|
|
387
|
+
try:
|
|
388
|
+
import importlib
|
|
389
|
+
|
|
390
|
+
module = importlib.import_module("tqdm.auto")
|
|
391
|
+
return getattr(module, "tqdm")
|
|
392
|
+
except Exception:
|
|
393
|
+
return None
|
|
394
|
+
|
|
395
|
+
@staticmethod
|
|
396
|
+
def _start_inline_status(label: str, enabled: bool) -> None:
|
|
397
|
+
"""Emit the leading inline progress label when requested (DRY helper)."""
|
|
398
|
+
|
|
399
|
+
if enabled:
|
|
400
|
+
print(label, end=" ", flush=True)
|
|
401
|
+
|
|
402
|
+
@staticmethod
|
|
403
|
+
def _render_inline_bar(pct: int, bar_len: int = 24) -> str:
|
|
404
|
+
"""Render a textual progress bar reused across inline progress sites."""
|
|
405
|
+
|
|
406
|
+
filled = (pct * bar_len) // 100
|
|
407
|
+
return "#" * filled + "-" * (bar_len - filled)
|
|
408
|
+
|
|
409
|
+
@classmethod
|
|
410
|
+
def _update_inline_status(
|
|
411
|
+
cls, label: str, pct: int, *, bar_len: int = 24, suffix: str = ""
|
|
412
|
+
) -> None:
|
|
413
|
+
"""Print an inline progress update with consistent formatting."""
|
|
414
|
+
|
|
415
|
+
bar = cls._render_inline_bar(pct, bar_len=bar_len)
|
|
416
|
+
suffix_text = f" {suffix}" if suffix else ""
|
|
417
|
+
print(f"\r{label} [{bar}] {pct:3d}%{suffix_text}", end="", flush=True)
|
|
418
|
+
|
|
419
|
+
@staticmethod
|
|
420
|
+
def _finish_inline_status(enabled: bool) -> None:
|
|
421
|
+
"""Terminate the inline status line so subsequent logs stay readable."""
|
|
422
|
+
|
|
423
|
+
if enabled:
|
|
424
|
+
print()
|
|
425
|
+
|
|
426
|
+
@staticmethod
|
|
427
|
+
def _pop_next_proposal(queue: List[RefactoringProposal]) -> Optional[RefactoringProposal]:
|
|
428
|
+
"""Remove and return the oldest queued proposal (DRY helper; see docs/DRY_RUN_2025-11-28.md)."""
|
|
429
|
+
|
|
430
|
+
if not queue:
|
|
431
|
+
return None
|
|
432
|
+
return queue.pop(0)
|
|
433
|
+
|
|
434
|
+
def _resolve_progress_backend(self, progress: str) -> Tuple[str, Optional[Any], bool]:
|
|
435
|
+
"""Normalize progress flag and load tqdm when available (mirrors DRY helper guidance)."""
|
|
436
|
+
|
|
437
|
+
allowed = {"auto", "tqdm", "none", "detail"}
|
|
438
|
+
normalized = progress if progress in allowed else "tqdm"
|
|
439
|
+
use_tqdm = normalized in {"auto", "tqdm"}
|
|
440
|
+
tqdm_cls: Optional[Any] = None
|
|
441
|
+
if use_tqdm:
|
|
442
|
+
tqdm_cls = self._load_tqdm_wrapper()
|
|
443
|
+
use_tqdm = tqdm_cls is not None
|
|
444
|
+
return normalized, tqdm_cls, use_tqdm
|
|
445
|
+
|
|
446
|
+
@staticmethod
|
|
447
|
+
def _function_contains_nonlocal(func: FunctionNode) -> bool:
|
|
448
|
+
"""Return True if the function body contains any nonlocal declarations."""
|
|
449
|
+
|
|
450
|
+
for stmt in func.body:
|
|
451
|
+
# Skip nested definitions; only consider nonlocal statements that belong
|
|
452
|
+
# to the function itself. Nonlocals inside nested functions do not impact
|
|
453
|
+
# whether the outer function may be safely extracted.
|
|
454
|
+
if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
455
|
+
continue
|
|
456
|
+
for node in ast.walk(stmt):
|
|
457
|
+
if isinstance(node, ast.Nonlocal):
|
|
458
|
+
return True
|
|
459
|
+
return False
|
|
460
|
+
|
|
461
|
+
@staticmethod
|
|
462
|
+
def _decorator_name(decorator: ast.expr) -> Optional[str]:
|
|
463
|
+
"""Return the simple name for a decorator expression if it can be resolved."""
|
|
464
|
+
|
|
465
|
+
if isinstance(decorator, ast.Name):
|
|
466
|
+
return decorator.id
|
|
467
|
+
if isinstance(decorator, ast.Attribute):
|
|
468
|
+
return decorator.attr
|
|
469
|
+
if isinstance(decorator, ast.Call):
|
|
470
|
+
return UnificationRefactorEngine._decorator_name(decorator.func)
|
|
471
|
+
return None
|
|
472
|
+
|
|
473
|
+
@staticmethod
|
|
474
|
+
def _has_decorator(fn: ast.FunctionDef, name: str) -> bool:
|
|
475
|
+
"""Return True when the function already carries a decorator with the given name."""
|
|
476
|
+
|
|
477
|
+
return any(
|
|
478
|
+
UnificationRefactorEngine._decorator_name(dec) == name for dec in fn.decorator_list
|
|
479
|
+
)
|
|
480
|
+
|
|
481
|
+
@staticmethod
|
|
482
|
+
def _strip_decorator(fn: ast.FunctionDef, name: str) -> None:
|
|
483
|
+
"""Remove any decorator whose resolved name matches ``name``."""
|
|
484
|
+
|
|
485
|
+
fn.decorator_list = [
|
|
486
|
+
dec
|
|
487
|
+
for dec in fn.decorator_list
|
|
488
|
+
if UnificationRefactorEngine._decorator_name(dec) != name
|
|
489
|
+
]
|
|
490
|
+
|
|
491
|
+
@staticmethod
|
|
492
|
+
def _ensure_leading_param(fn: ast.FunctionDef, param_name: str) -> None:
|
|
493
|
+
"""Ensure the positional-args list starts with ``param_name`` (preserving annotations)."""
|
|
494
|
+
|
|
495
|
+
existing: Optional[ast.arg] = None
|
|
496
|
+
remaining: List[ast.arg] = []
|
|
497
|
+
for arg in fn.args.args:
|
|
498
|
+
if arg.arg == param_name and existing is None:
|
|
499
|
+
existing = arg
|
|
500
|
+
continue
|
|
501
|
+
if arg.arg == param_name:
|
|
502
|
+
# Drop duplicate occurrences beyond the first
|
|
503
|
+
continue
|
|
504
|
+
remaining.append(arg)
|
|
505
|
+
|
|
506
|
+
if existing is None:
|
|
507
|
+
existing = ast.arg(arg=param_name)
|
|
508
|
+
|
|
509
|
+
fn.args.args = [existing] + remaining
|
|
510
|
+
|
|
511
|
+
@staticmethod
|
|
512
|
+
def _retarget_helper_calls(node: ast.AST, original_name: str, final_name: str) -> ast.AST:
|
|
513
|
+
"""Rewrite helper call-sites when the extracted helper is renamed (DRY helper parity)."""
|
|
514
|
+
|
|
515
|
+
if original_name == final_name:
|
|
516
|
+
return node
|
|
517
|
+
|
|
518
|
+
class _CallRenamer(ast.NodeTransformer):
|
|
519
|
+
def __init__(self, old: str, new: str) -> None:
|
|
520
|
+
self.old = old
|
|
521
|
+
self.new = new
|
|
522
|
+
|
|
523
|
+
def visit_Call(
|
|
524
|
+
self, call: ast.Call
|
|
525
|
+
) -> ast.AST: # pragma: no cover - simple AST rewrite
|
|
526
|
+
updated = cast(ast.Call, self.generic_visit(call))
|
|
527
|
+
if isinstance(updated.func, ast.Name) and updated.func.id == self.old:
|
|
528
|
+
updated.func.id = self.new
|
|
529
|
+
return updated
|
|
530
|
+
|
|
531
|
+
return _CallRenamer(original_name, final_name).visit(node)
|
|
532
|
+
|
|
533
|
+
@staticmethod
|
|
534
|
+
def _scan_module_docstring_and_imports(lines: List[str]) -> Tuple[int, int]:
|
|
535
|
+
"""Return last import line and docstring boundary (DRY helper parity)."""
|
|
536
|
+
|
|
537
|
+
in_docstring = False
|
|
538
|
+
docstring_char: Optional[str] = None
|
|
539
|
+
last_import_line = 0
|
|
540
|
+
after_docstring = 0
|
|
541
|
+
|
|
542
|
+
for i, line in enumerate(lines):
|
|
543
|
+
stripped = line.strip()
|
|
544
|
+
|
|
545
|
+
if i == 0 and (stripped.startswith('"""') or stripped.startswith("'''")):
|
|
546
|
+
docstring_char = stripped[:3]
|
|
547
|
+
if stripped.count(docstring_char) < 2:
|
|
548
|
+
in_docstring = True
|
|
549
|
+
else:
|
|
550
|
+
after_docstring = i + 1
|
|
551
|
+
continue
|
|
552
|
+
|
|
553
|
+
if in_docstring:
|
|
554
|
+
assert docstring_char is not None
|
|
555
|
+
if docstring_char in stripped:
|
|
556
|
+
in_docstring = False
|
|
557
|
+
after_docstring = i + 1
|
|
558
|
+
continue
|
|
559
|
+
|
|
560
|
+
if stripped.startswith("import ") or stripped.startswith("from "):
|
|
561
|
+
last_import_line = i + 1
|
|
562
|
+
continue
|
|
563
|
+
|
|
564
|
+
if last_import_line > 0 and stripped and not stripped.startswith("#"):
|
|
565
|
+
break
|
|
566
|
+
|
|
567
|
+
return last_import_line, after_docstring
|
|
568
|
+
|
|
569
|
+
def _allocate_helper_name(self, file_path: str) -> str:
|
|
570
|
+
"""Return a unique helper name for the given canonical file."""
|
|
571
|
+
|
|
572
|
+
counter = self._helper_name_counters.get(file_path)
|
|
573
|
+
if counter is None:
|
|
574
|
+
counter = self._discover_helper_counter_seed(file_path)
|
|
575
|
+
helper_name = f"__extracted_func_{counter}"
|
|
576
|
+
self._helper_name_counters[file_path] = counter + 1
|
|
577
|
+
return helper_name
|
|
578
|
+
|
|
579
|
+
def _discover_helper_counter_seed(self, file_path: str) -> int:
|
|
580
|
+
"""Prime the helper counter based on existing helper names in a file."""
|
|
581
|
+
|
|
582
|
+
try:
|
|
583
|
+
content = Path(file_path).read_text(encoding="utf-8")
|
|
584
|
+
except Exception:
|
|
585
|
+
return 0
|
|
586
|
+
|
|
587
|
+
pattern = re.compile(r"__extracted_func(?:_(\d+))?")
|
|
588
|
+
max_seen = -1
|
|
589
|
+
for match in pattern.finditer(content):
|
|
590
|
+
suffix = match.group(1)
|
|
591
|
+
idx = int(suffix) if suffix is not None else 0
|
|
592
|
+
if idx > max_seen:
|
|
593
|
+
max_seen = idx
|
|
594
|
+
return max_seen + 1
|
|
595
|
+
|
|
596
|
+
def _prepare_extracted_method_signature(
|
|
597
|
+
self,
|
|
598
|
+
fn: ast.FunctionDef,
|
|
599
|
+
method_kind: Literal["instance", "classmethod", "staticmethod"],
|
|
600
|
+
implicit_param: Optional[str],
|
|
601
|
+
) -> None:
|
|
602
|
+
"""Normalize the extracted helper so it behaves like the requested method type."""
|
|
603
|
+
|
|
604
|
+
if method_kind == "instance":
|
|
605
|
+
name = implicit_param or "self"
|
|
606
|
+
self._ensure_leading_param(fn, name)
|
|
607
|
+
# Strip any conflicting decorators that might have been synthesized earlier
|
|
608
|
+
self._strip_decorator(fn, "staticmethod")
|
|
609
|
+
self._strip_decorator(fn, "classmethod")
|
|
610
|
+
elif method_kind == "classmethod":
|
|
611
|
+
name = implicit_param or "cls"
|
|
612
|
+
self._ensure_leading_param(fn, name)
|
|
613
|
+
self._strip_decorator(fn, "staticmethod")
|
|
614
|
+
if not self._has_decorator(fn, "classmethod"):
|
|
615
|
+
fn.decorator_list.insert(0, ast.Name(id="classmethod", ctx=ast.Load()))
|
|
616
|
+
elif method_kind == "staticmethod":
|
|
617
|
+
self._strip_decorator(fn, "classmethod")
|
|
618
|
+
if not self._has_decorator(fn, "staticmethod"):
|
|
619
|
+
fn.decorator_list.insert(0, ast.Name(id="staticmethod", ctx=ast.Load()))
|
|
620
|
+
else:
|
|
621
|
+
raise ValueError(f"Unsupported method kind: {method_kind}")
|
|
622
|
+
|
|
623
|
+
def _rewrite_call_for_method(
|
|
624
|
+
self,
|
|
625
|
+
node: ast.AST,
|
|
626
|
+
original_name: str,
|
|
627
|
+
new_name: str,
|
|
628
|
+
method_kind: Optional[Literal["instance", "classmethod", "staticmethod"]],
|
|
629
|
+
implicit_param: Optional[str],
|
|
630
|
+
class_name: Optional[str],
|
|
631
|
+
) -> ast.AST:
|
|
632
|
+
"""Rewrite calls to the extracted helper so they use method dispatch semantics."""
|
|
633
|
+
|
|
634
|
+
if method_kind is None:
|
|
635
|
+
return node
|
|
636
|
+
|
|
637
|
+
rewriter = MethodCallRewriter(
|
|
638
|
+
self._drop_implicit_positional,
|
|
639
|
+
self._drop_implicit_keyword,
|
|
640
|
+
original_name=original_name,
|
|
641
|
+
new_name=new_name,
|
|
642
|
+
method_kind=method_kind,
|
|
643
|
+
implicit_name=implicit_param,
|
|
644
|
+
class_name=class_name,
|
|
645
|
+
)
|
|
646
|
+
return rewriter.visit(node) # type: ignore[no-any-return]
|
|
647
|
+
|
|
648
|
+
@staticmethod
|
|
649
|
+
def _drop_implicit_positional(args: List[ast.expr], implicit_name: str) -> List[ast.expr]:
|
|
650
|
+
"""Drop the first positional argument matching ``implicit_name`` if present."""
|
|
651
|
+
|
|
652
|
+
result: List[ast.expr] = []
|
|
653
|
+
dropped = False
|
|
654
|
+
for arg in args:
|
|
655
|
+
if not dropped and isinstance(arg, ast.Name) and arg.id == implicit_name:
|
|
656
|
+
dropped = True
|
|
657
|
+
continue
|
|
658
|
+
result.append(arg)
|
|
659
|
+
return result
|
|
660
|
+
|
|
661
|
+
@staticmethod
|
|
662
|
+
def _drop_implicit_keyword(
|
|
663
|
+
keywords: List[ast.keyword], implicit_name: str
|
|
664
|
+
) -> List[ast.keyword]:
|
|
665
|
+
"""Drop the first keyword argument whose name matches ``implicit_name``."""
|
|
666
|
+
|
|
667
|
+
result: List[ast.keyword] = []
|
|
668
|
+
dropped = False
|
|
669
|
+
for kw in keywords:
|
|
670
|
+
if not dropped and kw.arg == implicit_name:
|
|
671
|
+
dropped = True
|
|
672
|
+
continue
|
|
673
|
+
result.append(kw)
|
|
674
|
+
return result
|
|
675
|
+
|
|
676
|
+
def _get_method_context(
|
|
677
|
+
self, func: Optional[FunctionNode], class_name: Optional[str]
|
|
678
|
+
) -> MethodInfo:
|
|
679
|
+
"""Return method metadata for ``func`` when it is defined inside ``class_name``."""
|
|
680
|
+
|
|
681
|
+
if func is None or class_name is None:
|
|
682
|
+
return MethodInfo(kind=None, implicit_param=None)
|
|
683
|
+
|
|
684
|
+
kind: Optional[Literal["instance", "classmethod", "staticmethod"]] = None
|
|
685
|
+
for decorator in func.decorator_list:
|
|
686
|
+
name = self._decorator_name(decorator)
|
|
687
|
+
if name == "staticmethod":
|
|
688
|
+
kind = "staticmethod"
|
|
689
|
+
break
|
|
690
|
+
if name == "classmethod":
|
|
691
|
+
kind = "classmethod"
|
|
692
|
+
break
|
|
693
|
+
|
|
694
|
+
if kind is None:
|
|
695
|
+
kind = "instance"
|
|
696
|
+
|
|
697
|
+
implicit_param = None
|
|
698
|
+
if kind in {"instance", "classmethod"}:
|
|
699
|
+
if func.args.args:
|
|
700
|
+
implicit_param = func.args.args[0].arg
|
|
701
|
+
else:
|
|
702
|
+
implicit_param = "self" if kind == "instance" else "cls"
|
|
703
|
+
|
|
704
|
+
return MethodInfo(kind=kind, implicit_param=implicit_param)
|
|
705
|
+
|
|
706
|
+
@staticmethod
|
|
707
|
+
def _resolve_base_name(expr: ast.expr) -> Optional[str]:
|
|
708
|
+
"""Resolve a base-class expression into its dotted name when feasible."""
|
|
709
|
+
|
|
710
|
+
if isinstance(expr, ast.Name):
|
|
711
|
+
return expr.id
|
|
712
|
+
if isinstance(expr, ast.Attribute):
|
|
713
|
+
parts: List[str] = []
|
|
714
|
+
current: ast.expr = expr
|
|
715
|
+
while isinstance(current, ast.Attribute):
|
|
716
|
+
parts.append(current.attr)
|
|
717
|
+
current = current.value
|
|
718
|
+
if isinstance(current, ast.Name):
|
|
719
|
+
parts.append(current.id)
|
|
720
|
+
return ".".join(reversed(parts))
|
|
721
|
+
return None
|
|
722
|
+
|
|
723
|
+
@staticmethod
|
|
724
|
+
def _class_info_key(info: ClassInfo) -> Tuple[str, str]:
|
|
725
|
+
"""Return a stable identifier for a class definition."""
|
|
726
|
+
|
|
727
|
+
return (info.file_path, info.qualname)
|
|
728
|
+
|
|
729
|
+
def _find_class_info_by_name(
|
|
730
|
+
self, class_infos: List[ClassInfo], file_path: str, class_name: str
|
|
731
|
+
) -> Optional[ClassInfo]:
|
|
732
|
+
"""Locate class metadata using its defining file and simple name."""
|
|
733
|
+
|
|
734
|
+
matches = [
|
|
735
|
+
info for info in class_infos if info.file_path == file_path and info.name == class_name
|
|
736
|
+
]
|
|
737
|
+
if not matches:
|
|
738
|
+
return None
|
|
739
|
+
if len(matches) == 1:
|
|
740
|
+
return matches[0]
|
|
741
|
+
|
|
742
|
+
# Prefer the innermost definition (longest qualname) when duplicates exist.
|
|
743
|
+
matches.sort(key=lambda info: info.qualname.count("."), reverse=True)
|
|
744
|
+
return matches[0]
|
|
745
|
+
|
|
746
|
+
def _find_class_info_for_base(
|
|
747
|
+
self,
|
|
748
|
+
class_infos: List[ClassInfo],
|
|
749
|
+
base_name: str,
|
|
750
|
+
*,
|
|
751
|
+
prefer_file: Optional[str] = None,
|
|
752
|
+
) -> Optional[ClassInfo]:
|
|
753
|
+
"""Resolve a base-class reference to known class metadata when possible."""
|
|
754
|
+
|
|
755
|
+
# Exact qualname match first (covers nested classes written as Outer.Inner)
|
|
756
|
+
qual_matches = [info for info in class_infos if info.qualname == base_name]
|
|
757
|
+
if prefer_file is not None:
|
|
758
|
+
for info in qual_matches:
|
|
759
|
+
if info.file_path == prefer_file:
|
|
760
|
+
return info
|
|
761
|
+
if qual_matches:
|
|
762
|
+
return qual_matches[0]
|
|
763
|
+
|
|
764
|
+
simple_name = base_name.split(".")[-1]
|
|
765
|
+
simple_matches = [info for info in class_infos if info.name == simple_name]
|
|
766
|
+
if prefer_file is not None:
|
|
767
|
+
for info in simple_matches:
|
|
768
|
+
if info.file_path == prefer_file:
|
|
769
|
+
return info
|
|
770
|
+
if len(simple_matches) == 1:
|
|
771
|
+
return simple_matches[0]
|
|
772
|
+
return None
|
|
773
|
+
|
|
774
|
+
def _collect_class_ancestors(
|
|
775
|
+
self, class_info: ClassInfo, class_infos: List[ClassInfo]
|
|
776
|
+
) -> List[ClassInfo]:
|
|
777
|
+
"""Return ancestors starting from the nearest base class."""
|
|
778
|
+
|
|
779
|
+
ancestors: List[Tuple[int, ClassInfo]] = []
|
|
780
|
+
visited: Set[Tuple[str, str]] = set()
|
|
781
|
+
queue: deque[Tuple[ClassInfo, int]] = deque([(class_info, 0)])
|
|
782
|
+
|
|
783
|
+
while queue:
|
|
784
|
+
current, depth = queue.popleft()
|
|
785
|
+
for base_name in current.bases:
|
|
786
|
+
base_info = self._find_class_info_for_base(
|
|
787
|
+
class_infos, base_name, prefer_file=current.file_path
|
|
788
|
+
)
|
|
789
|
+
if base_info is None:
|
|
790
|
+
continue
|
|
791
|
+
key = self._class_info_key(base_info)
|
|
792
|
+
if key in visited:
|
|
793
|
+
continue
|
|
794
|
+
visited.add(key)
|
|
795
|
+
ancestors.append((depth + 1, base_info))
|
|
796
|
+
queue.append((base_info, depth + 1))
|
|
797
|
+
|
|
798
|
+
ancestors.sort(key=lambda item: item[0])
|
|
799
|
+
return [info for _depth, info in ancestors]
|
|
800
|
+
|
|
801
|
+
def _find_common_ancestor(
|
|
802
|
+
self,
|
|
803
|
+
class1: Tuple[str, str],
|
|
804
|
+
class2: Tuple[str, str],
|
|
805
|
+
class_infos: List[ClassInfo],
|
|
806
|
+
) -> Optional[ClassInfo]:
|
|
807
|
+
"""Return the nearest shared ancestor class for two class definitions."""
|
|
808
|
+
|
|
809
|
+
file1, name1 = class1
|
|
810
|
+
file2, name2 = class2
|
|
811
|
+
|
|
812
|
+
info1 = self._find_class_info_by_name(class_infos, file1, name1)
|
|
813
|
+
info2 = self._find_class_info_by_name(class_infos, file2, name2)
|
|
814
|
+
if info1 is None or info2 is None:
|
|
815
|
+
return None
|
|
816
|
+
|
|
817
|
+
key1 = self._class_info_key(info1)
|
|
818
|
+
key2 = self._class_info_key(info2)
|
|
819
|
+
|
|
820
|
+
chain1 = [info1] + self._collect_class_ancestors(info1, class_infos)
|
|
821
|
+
chain2 = [info2] + self._collect_class_ancestors(info2, class_infos)
|
|
822
|
+
lookup2 = {self._class_info_key(info): info for info in chain2}
|
|
823
|
+
|
|
824
|
+
for info in chain1:
|
|
825
|
+
key = self._class_info_key(info)
|
|
826
|
+
if key in lookup2:
|
|
827
|
+
if key == key1 and key == key2:
|
|
828
|
+
# Identical class; handled elsewhere.
|
|
829
|
+
continue
|
|
830
|
+
return lookup2[key]
|
|
831
|
+
return None
|
|
832
|
+
|
|
833
|
+
def _choose_class_insertion(
|
|
834
|
+
self,
|
|
835
|
+
pair: CodeBlockPair,
|
|
836
|
+
method_info1: MethodInfo,
|
|
837
|
+
method_info2: MethodInfo,
|
|
838
|
+
class_infos: List[ClassInfo],
|
|
839
|
+
) -> Optional[ClassInsertionPlan]:
|
|
840
|
+
"""Determine whether the helper should be inserted into a class context."""
|
|
841
|
+
# Original logic required both method kinds to be equal and non-None. In practice
|
|
842
|
+
# some methods may not have their kind inferred (kind=None) even though they are
|
|
843
|
+
# instance methods (implicit_param present / first arg named 'self'). We relax
|
|
844
|
+
# this by attempting inference when BOTH kinds are None; if only one side is
|
|
845
|
+
# None we keep the conservative skip to avoid mismatches (e.g., staticmethod
|
|
846
|
+
# vs instance).
|
|
847
|
+
k1, k2 = method_info1.kind, method_info2.kind
|
|
848
|
+
inferred_kind: Optional[Literal["instance", "classmethod", "staticmethod"]] = None
|
|
849
|
+
if k1 is None and k2 is None:
|
|
850
|
+
# Infer 'instance' if both have an implicit_param or a first argument name.
|
|
851
|
+
# method_info.implicit_param is populated for recognized methods; as a
|
|
852
|
+
# fallback, treat absence uniformly as instance.
|
|
853
|
+
inferred_kind = "instance"
|
|
854
|
+
else:
|
|
855
|
+
if k1 != k2 or k1 is None or k2 is None:
|
|
856
|
+
return None
|
|
857
|
+
if pair.class1_name is None or pair.class2_name is None:
|
|
858
|
+
return None
|
|
859
|
+
|
|
860
|
+
file1 = pair.file_path
|
|
861
|
+
file2 = pair.file_path2 or pair.file_path
|
|
862
|
+
|
|
863
|
+
# At this point we know effective_kind is valid because we've already validated k1/k2
|
|
864
|
+
effective_kind: Literal["instance", "classmethod", "staticmethod"] = cast(
|
|
865
|
+
Literal["instance", "classmethod", "staticmethod"], inferred_kind or method_info1.kind
|
|
866
|
+
)
|
|
867
|
+
if file1 == file2 and pair.class1_name == pair.class2_name:
|
|
868
|
+
implicit_param = method_info1.implicit_param or method_info2.implicit_param
|
|
869
|
+
if effective_kind == "instance" and not implicit_param:
|
|
870
|
+
implicit_param = "self"
|
|
871
|
+
if effective_kind == "classmethod" and not implicit_param:
|
|
872
|
+
implicit_param = "cls"
|
|
873
|
+
return ClassInsertionPlan(
|
|
874
|
+
class_name=pair.class1_name,
|
|
875
|
+
file_path=file1,
|
|
876
|
+
method_kind=effective_kind,
|
|
877
|
+
implicit_param=implicit_param,
|
|
878
|
+
)
|
|
879
|
+
|
|
880
|
+
ancestor = self._find_common_ancestor(
|
|
881
|
+
(file1, pair.class1_name),
|
|
882
|
+
(file2, pair.class2_name),
|
|
883
|
+
class_infos,
|
|
884
|
+
)
|
|
885
|
+
if ancestor is None:
|
|
886
|
+
return None
|
|
887
|
+
|
|
888
|
+
implicit_param = method_info1.implicit_param or method_info2.implicit_param
|
|
889
|
+
if effective_kind == "instance" and not implicit_param:
|
|
890
|
+
implicit_param = "self"
|
|
891
|
+
if effective_kind == "classmethod" and not implicit_param:
|
|
892
|
+
implicit_param = "cls"
|
|
893
|
+
|
|
894
|
+
return ClassInsertionPlan(
|
|
895
|
+
class_name=ancestor.name,
|
|
896
|
+
file_path=ancestor.file_path,
|
|
897
|
+
method_kind=effective_kind,
|
|
898
|
+
implicit_param=implicit_param,
|
|
899
|
+
)
|
|
900
|
+
|
|
901
|
+
def _evaluate_pairs_serial(
|
|
902
|
+
self,
|
|
903
|
+
block_pairs: List[CodeBlockPair],
|
|
904
|
+
all_functions: List[
|
|
905
|
+
Tuple[
|
|
906
|
+
str,
|
|
907
|
+
FunctionNode,
|
|
908
|
+
str,
|
|
909
|
+
ScopeAnalyzer,
|
|
910
|
+
Scope,
|
|
911
|
+
Optional[str],
|
|
912
|
+
Optional[str],
|
|
913
|
+
List[str],
|
|
914
|
+
]
|
|
915
|
+
],
|
|
916
|
+
class_infos: List[ClassInfo],
|
|
917
|
+
*,
|
|
918
|
+
verbose: bool,
|
|
919
|
+
progress: str,
|
|
920
|
+
) -> List[RefactoringProposal]:
|
|
921
|
+
proposals: List[RefactoringProposal] = []
|
|
922
|
+
|
|
923
|
+
progress_mode, tqdm_cls, use_tqdm = self._resolve_progress_backend(progress)
|
|
924
|
+
tqdm_iter = None
|
|
925
|
+
# Always show progress for pair evaluation when progress is enabled, even if verbose=False
|
|
926
|
+
if use_tqdm and tqdm_cls is not None:
|
|
927
|
+
tqdm_iter = tqdm_cls(
|
|
928
|
+
block_pairs,
|
|
929
|
+
total=len(block_pairs),
|
|
930
|
+
desc="unify",
|
|
931
|
+
unit="pair",
|
|
932
|
+
leave=False,
|
|
933
|
+
)
|
|
934
|
+
|
|
935
|
+
if use_tqdm and tqdm_iter is not None:
|
|
936
|
+
for pair in tqdm_iter:
|
|
937
|
+
proposal = self._try_refactor_pair_multi_file(pair, all_functions, class_infos)
|
|
938
|
+
if proposal:
|
|
939
|
+
proposals.append(proposal)
|
|
940
|
+
return proposals
|
|
941
|
+
|
|
942
|
+
use_inline_bar = progress_mode in ("auto", "tqdm") and len(block_pairs) > 0 and not use_tqdm
|
|
943
|
+
last_pct = -1
|
|
944
|
+
self._start_inline_status("Analyzing pairs (unify):", use_inline_bar)
|
|
945
|
+
|
|
946
|
+
for idx, pair in enumerate(block_pairs, 1):
|
|
947
|
+
proposal = self._try_refactor_pair_multi_file(pair, all_functions, class_infos)
|
|
948
|
+
if proposal:
|
|
949
|
+
proposals.append(proposal)
|
|
950
|
+
if use_inline_bar:
|
|
951
|
+
pct = int(100 * idx / len(block_pairs))
|
|
952
|
+
if pct != last_pct:
|
|
953
|
+
last_pct = pct
|
|
954
|
+
self._update_inline_status("Analyzing pairs (unify):", pct)
|
|
955
|
+
|
|
956
|
+
self._finish_inline_status(use_inline_bar)
|
|
957
|
+
|
|
958
|
+
return proposals
|
|
959
|
+
|
|
960
|
+
def _evaluate_pairs_parallel(
|
|
961
|
+
self,
|
|
962
|
+
block_pairs: List[CodeBlockPair],
|
|
963
|
+
all_functions: List[
|
|
964
|
+
Tuple[
|
|
965
|
+
str,
|
|
966
|
+
FunctionNode,
|
|
967
|
+
str,
|
|
968
|
+
ScopeAnalyzer,
|
|
969
|
+
Scope,
|
|
970
|
+
Optional[str],
|
|
971
|
+
Optional[str],
|
|
972
|
+
List[str],
|
|
973
|
+
]
|
|
974
|
+
],
|
|
975
|
+
class_infos: List[ClassInfo],
|
|
976
|
+
*,
|
|
977
|
+
verbose: bool,
|
|
978
|
+
progress: str,
|
|
979
|
+
) -> List[RefactoringProposal]:
|
|
980
|
+
progress_mode, tqdm_cls, tqdm_available = self._resolve_progress_backend(progress)
|
|
981
|
+
|
|
982
|
+
pair_count = len(block_pairs)
|
|
983
|
+
cpu_count = os.cpu_count() or 1
|
|
984
|
+
max_workers = min(cpu_count, pair_count)
|
|
985
|
+
if max_workers <= 1:
|
|
986
|
+
return self._evaluate_pairs_serial(
|
|
987
|
+
block_pairs,
|
|
988
|
+
all_functions,
|
|
989
|
+
class_infos,
|
|
990
|
+
verbose=verbose,
|
|
991
|
+
progress=progress,
|
|
992
|
+
)
|
|
993
|
+
|
|
994
|
+
engine_config: Dict[str, Optional[object]] = {
|
|
995
|
+
"max_parameters": self.max_parameters,
|
|
996
|
+
"min_lines": self.min_lines,
|
|
997
|
+
"parameterize_constants": self.parameterize_constants,
|
|
998
|
+
"prefer_absolute_imports": self.prefer_absolute_imports,
|
|
999
|
+
"pep420_namespace_packages": self.pep420_namespace_packages,
|
|
1000
|
+
}
|
|
1001
|
+
|
|
1002
|
+
tasks = [(idx, pair) for idx, pair in enumerate(block_pairs)]
|
|
1003
|
+
ordered_results: List[Optional[RefactoringProposal]] = [None] * pair_count
|
|
1004
|
+
|
|
1005
|
+
use_tqdm = verbose and tqdm_available and tqdm_cls is not None
|
|
1006
|
+
tqdm_wrapper = tqdm_cls if use_tqdm else None
|
|
1007
|
+
|
|
1008
|
+
use_inline_bar = verbose and not use_tqdm and progress_mode in ("auto", "tqdm")
|
|
1009
|
+
last_pct = -1
|
|
1010
|
+
completed = 0
|
|
1011
|
+
self._start_inline_status("Analyzing candidate pairs:", use_inline_bar)
|
|
1012
|
+
|
|
1013
|
+
# ProcessPoolExecutor type stubs are restrictive; cast to Any for initializer/initargs
|
|
1014
|
+
executor_kwargs: Dict[str, Any] = {
|
|
1015
|
+
"max_workers": max_workers,
|
|
1016
|
+
"initializer": _initialize_pair_worker,
|
|
1017
|
+
"initargs": (engine_config, all_functions, class_infos),
|
|
1018
|
+
}
|
|
1019
|
+
with ProcessPoolExecutor(**executor_kwargs) as executor:
|
|
1020
|
+
futures = [executor.submit(_process_pair_in_worker, task) for task in tasks]
|
|
1021
|
+
iterator = as_completed(futures)
|
|
1022
|
+
if use_tqdm and tqdm_wrapper is not None:
|
|
1023
|
+
iterator = tqdm_wrapper(
|
|
1024
|
+
iterator,
|
|
1025
|
+
total=pair_count,
|
|
1026
|
+
desc="Analyzing candidate pairs",
|
|
1027
|
+
unit="pair",
|
|
1028
|
+
leave=False,
|
|
1029
|
+
)
|
|
1030
|
+
|
|
1031
|
+
for future in iterator:
|
|
1032
|
+
pair_index, proposal = future.result()
|
|
1033
|
+
if proposal:
|
|
1034
|
+
ordered_results[pair_index] = proposal
|
|
1035
|
+
completed += 1
|
|
1036
|
+
if use_inline_bar:
|
|
1037
|
+
pct = int(100 * completed / pair_count)
|
|
1038
|
+
if pct != last_pct:
|
|
1039
|
+
last_pct = pct
|
|
1040
|
+
self._update_inline_status("Analyzing candidate pairs:", pct)
|
|
1041
|
+
|
|
1042
|
+
self._finish_inline_status(use_inline_bar)
|
|
1043
|
+
|
|
1044
|
+
return [proposal for proposal in ordered_results if proposal is not None]
|
|
1045
|
+
|
|
1046
|
+
@staticmethod
|
|
1047
|
+
def _block_line_span(block: Sequence[ast.stmt]) -> Optional[Tuple[int, int]]:
|
|
1048
|
+
"""Return the (start_line, end_line) span for a contiguous block of statements."""
|
|
1049
|
+
|
|
1050
|
+
if not block:
|
|
1051
|
+
return None
|
|
1052
|
+
|
|
1053
|
+
start_node = block[0]
|
|
1054
|
+
end_node = block[-1]
|
|
1055
|
+
|
|
1056
|
+
start_line = getattr(start_node, "lineno", None)
|
|
1057
|
+
end_line = getattr(end_node, "end_lineno", None) or getattr(end_node, "lineno", None)
|
|
1058
|
+
if start_line is None or end_line is None:
|
|
1059
|
+
return None
|
|
1060
|
+
|
|
1061
|
+
return int(start_line), int(end_line)
|
|
1062
|
+
|
|
1063
|
+
@staticmethod
|
|
1064
|
+
def _body_without_docstring(body: Sequence[ast.stmt]) -> List[ast.stmt]:
|
|
1065
|
+
"""Return body statements with a leading docstring removed when present."""
|
|
1066
|
+
|
|
1067
|
+
body_list = list(body)
|
|
1068
|
+
if not body_list:
|
|
1069
|
+
return []
|
|
1070
|
+
|
|
1071
|
+
first_stmt = body_list[0]
|
|
1072
|
+
if (
|
|
1073
|
+
isinstance(first_stmt, ast.Expr)
|
|
1074
|
+
and isinstance(first_stmt.value, ast.Constant)
|
|
1075
|
+
and isinstance(first_stmt.value.value, str)
|
|
1076
|
+
):
|
|
1077
|
+
return body_list[1:]
|
|
1078
|
+
|
|
1079
|
+
return body_list
|
|
1080
|
+
|
|
1081
|
+
def _extract_code_blocks(
|
|
1082
|
+
self, function: FunctionNode
|
|
1083
|
+
) -> List[Tuple[Tuple[int, int], List[ast.AST]]]:
|
|
1084
|
+
"""
|
|
1085
|
+
Extract all contiguous code blocks from a function body, including nested bodies.
|
|
1086
|
+
|
|
1087
|
+
Args:
|
|
1088
|
+
function: Function definition
|
|
1089
|
+
|
|
1090
|
+
Returns:
|
|
1091
|
+
List of (line_range, statements) tuples
|
|
1092
|
+
"""
|
|
1093
|
+
|
|
1094
|
+
def extract_from_body(body: List[ast.stmt]) -> List[Tuple[Tuple[int, int], List[ast.AST]]]:
|
|
1095
|
+
# Extract all contiguous subsequences of minimum length from a given body
|
|
1096
|
+
results: List[Tuple[Tuple[int, int], List[ast.AST]]] = []
|
|
1097
|
+
|
|
1098
|
+
# Extract all contiguous subsequences
|
|
1099
|
+
for length in range(len(body), 0, -1):
|
|
1100
|
+
for start in range(len(body) - length + 1):
|
|
1101
|
+
block = body[start : start + length]
|
|
1102
|
+
|
|
1103
|
+
# Do not extract blocks that contain nested function/class definitions.
|
|
1104
|
+
# These statements establish new scopes whose bindings must remain in the
|
|
1105
|
+
# original function so later statements can reference them.
|
|
1106
|
+
if any(
|
|
1107
|
+
isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))
|
|
1108
|
+
for stmt in block
|
|
1109
|
+
):
|
|
1110
|
+
continue
|
|
1111
|
+
|
|
1112
|
+
span = self._block_line_span(block)
|
|
1113
|
+
if span is None:
|
|
1114
|
+
continue
|
|
1115
|
+
start_line, end_line = span
|
|
1116
|
+
line_count = end_line - start_line + 1
|
|
1117
|
+
|
|
1118
|
+
if line_count >= self.min_lines:
|
|
1119
|
+
results.append(((start_line, end_line), cast(List[ast.AST], block)))
|
|
1120
|
+
|
|
1121
|
+
# Recurse into nested bodies for control-flow/container statements
|
|
1122
|
+
for stmt in body:
|
|
1123
|
+
# Skip nested function/class definitions to avoid crossing scopes
|
|
1124
|
+
if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
|
|
1125
|
+
continue
|
|
1126
|
+
|
|
1127
|
+
# Common body/orelse containers
|
|
1128
|
+
if hasattr(stmt, "body") and isinstance(getattr(stmt, "body"), list):
|
|
1129
|
+
results.extend(extract_from_body(getattr(stmt, "body")))
|
|
1130
|
+
if hasattr(stmt, "orelse") and isinstance(getattr(stmt, "orelse"), list):
|
|
1131
|
+
results.extend(extract_from_body(getattr(stmt, "orelse")))
|
|
1132
|
+
|
|
1133
|
+
# With and AsyncWith already covered by .body
|
|
1134
|
+
# Try/Except/Finally blocks
|
|
1135
|
+
if isinstance(stmt, ast.Try):
|
|
1136
|
+
if stmt.handlers:
|
|
1137
|
+
for h in stmt.handlers:
|
|
1138
|
+
if hasattr(h, "body") and isinstance(h.body, list):
|
|
1139
|
+
results.extend(extract_from_body(h.body))
|
|
1140
|
+
if hasattr(stmt, "finalbody") and isinstance(stmt.finalbody, list):
|
|
1141
|
+
results.extend(extract_from_body(stmt.finalbody))
|
|
1142
|
+
|
|
1143
|
+
return results
|
|
1144
|
+
|
|
1145
|
+
# Prepare top-level body (skip docstring)
|
|
1146
|
+
body = self._body_without_docstring(function.body)
|
|
1147
|
+
|
|
1148
|
+
return extract_from_body(body)
|
|
1149
|
+
|
|
1150
|
+
def _has_code_after_block(self, function: ast.FunctionDef, block_end_line: int) -> bool:
|
|
1151
|
+
"""
|
|
1152
|
+
Check if there's any executable code after block_end_line in the function.
|
|
1153
|
+
|
|
1154
|
+
This is used to detect when extracting a block with returns would make
|
|
1155
|
+
subsequent code unreachable.
|
|
1156
|
+
|
|
1157
|
+
Args:
|
|
1158
|
+
function: Function definition
|
|
1159
|
+
block_end_line: End line of the block
|
|
1160
|
+
|
|
1161
|
+
Returns:
|
|
1162
|
+
True if there's code after the block
|
|
1163
|
+
"""
|
|
1164
|
+
body = self._body_without_docstring(function.body)
|
|
1165
|
+
|
|
1166
|
+
# Check if any statement starts after block_end_line
|
|
1167
|
+
for stmt in body:
|
|
1168
|
+
if stmt.lineno > block_end_line:
|
|
1169
|
+
return True
|
|
1170
|
+
return False
|
|
1171
|
+
|
|
1172
|
+
def _has_returns_in_loops(self, block: List[ast.AST]) -> bool:
|
|
1173
|
+
"""
|
|
1174
|
+
Check if a block contains return statements inside loops.
|
|
1175
|
+
|
|
1176
|
+
Returns in loops are conditional on the loop executing, so if the loop
|
|
1177
|
+
doesn't execute (e.g., empty iteration), control continues after the loop.
|
|
1178
|
+
|
|
1179
|
+
Args:
|
|
1180
|
+
block: List of AST statements
|
|
1181
|
+
|
|
1182
|
+
Returns:
|
|
1183
|
+
True if there are returns inside loop statements
|
|
1184
|
+
"""
|
|
1185
|
+
|
|
1186
|
+
finder = LoopReturnFinder()
|
|
1187
|
+
for stmt in block:
|
|
1188
|
+
finder.visit(stmt)
|
|
1189
|
+
return finder.has_loop_return
|
|
1190
|
+
|
|
1191
|
+
def _get_used_names(self, node: ast.AST) -> Set[str]:
|
|
1192
|
+
"""
|
|
1193
|
+
Get all variable names that are used (read from) in an AST node.
|
|
1194
|
+
|
|
1195
|
+
This collects all Name nodes with Load context.
|
|
1196
|
+
|
|
1197
|
+
Args:
|
|
1198
|
+
node: AST node to analyze
|
|
1199
|
+
|
|
1200
|
+
Returns:
|
|
1201
|
+
Set of variable names that are read in the node
|
|
1202
|
+
"""
|
|
1203
|
+
cached = self._used_names_cache.get(node)
|
|
1204
|
+
if cached is not None:
|
|
1205
|
+
return set(cached)
|
|
1206
|
+
|
|
1207
|
+
collector = NameCollector()
|
|
1208
|
+
collector.visit(node)
|
|
1209
|
+
|
|
1210
|
+
frozen = frozenset(collector.used)
|
|
1211
|
+
self._used_names_cache[node] = frozen
|
|
1212
|
+
return set(frozen)
|
|
1213
|
+
|
|
1214
|
+
def _collect_parameter_names(self, func: FunctionNode) -> Set[str]:
|
|
1215
|
+
"""Return all argument names for a function (including pos-only and varargs)."""
|
|
1216
|
+
|
|
1217
|
+
if not isinstance(func, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
|
1218
|
+
return set()
|
|
1219
|
+
|
|
1220
|
+
params: Set[str] = set()
|
|
1221
|
+
args = func.args
|
|
1222
|
+
|
|
1223
|
+
for arg in getattr(args, "posonlyargs", []) or []:
|
|
1224
|
+
params.add(arg.arg)
|
|
1225
|
+
for arg in args.args:
|
|
1226
|
+
params.add(arg.arg)
|
|
1227
|
+
for arg in args.kwonlyargs:
|
|
1228
|
+
params.add(arg.arg)
|
|
1229
|
+
if args.vararg:
|
|
1230
|
+
params.add(args.vararg.arg)
|
|
1231
|
+
if args.kwarg:
|
|
1232
|
+
params.add(args.kwarg.arg)
|
|
1233
|
+
|
|
1234
|
+
return params
|
|
1235
|
+
|
|
1236
|
+
def _deepest_common_ancestry(
|
|
1237
|
+
self, anc1: Optional[List[str]], anc2: Optional[List[str]]
|
|
1238
|
+
) -> Optional[str]:
|
|
1239
|
+
"""Return deepest shared symbol in two ancestry chains (DRY helper \u00a7ref docs/DRY_RUN_2025-11-28.md)."""
|
|
1240
|
+
|
|
1241
|
+
if not anc1 or not anc2:
|
|
1242
|
+
return None
|
|
1243
|
+
|
|
1244
|
+
dce: Optional[str] = None
|
|
1245
|
+
for left, right in zip(anc1, anc2):
|
|
1246
|
+
if left == right:
|
|
1247
|
+
dce = left
|
|
1248
|
+
else:
|
|
1249
|
+
break
|
|
1250
|
+
return dce
|
|
1251
|
+
|
|
1252
|
+
def _build_block_binding_snapshot(
|
|
1253
|
+
self,
|
|
1254
|
+
func: FunctionNode,
|
|
1255
|
+
block_nodes: List[ast.AST],
|
|
1256
|
+
block_range: Tuple[int, int],
|
|
1257
|
+
reassignments: Dict[int, bool],
|
|
1258
|
+
) -> BlockBindingSnapshot:
|
|
1259
|
+
"""Aggregate binding stats for a block (DRY helper import, see docs/DRY_RUN_2025-11-28.md)."""
|
|
1260
|
+
|
|
1261
|
+
bound_in_block, reassigned_in_block = _collect_block_binding_stats(
|
|
1262
|
+
block_nodes, reassignments
|
|
1263
|
+
)
|
|
1264
|
+
|
|
1265
|
+
bound_before_block: Set[str] = set()
|
|
1266
|
+
block_start_line = block_range[0]
|
|
1267
|
+
for stmt in func.body:
|
|
1268
|
+
if hasattr(stmt, "lineno") and stmt.lineno < block_start_line:
|
|
1269
|
+
stmt_bound: Set[str] = set()
|
|
1270
|
+
stmt_reassigned: Set[str] = set()
|
|
1271
|
+
_collect_bindings_and_reassignments(
|
|
1272
|
+
stmt, reassignments, stmt_bound, stmt_reassigned
|
|
1273
|
+
)
|
|
1274
|
+
bound_before_block.update(stmt_bound)
|
|
1275
|
+
|
|
1276
|
+
bound_before_block.update(self._collect_parameter_names(func))
|
|
1277
|
+
|
|
1278
|
+
bound_after_block: Set[str] = set()
|
|
1279
|
+
block_end_line = block_range[1]
|
|
1280
|
+
for stmt in func.body:
|
|
1281
|
+
if hasattr(stmt, "lineno") and stmt.lineno > block_end_line:
|
|
1282
|
+
stmt_bound = set()
|
|
1283
|
+
stmt_reassigned = set()
|
|
1284
|
+
_collect_bindings_and_reassignments(
|
|
1285
|
+
stmt, reassignments, stmt_bound, stmt_reassigned
|
|
1286
|
+
)
|
|
1287
|
+
bound_after_block.update(stmt_bound)
|
|
1288
|
+
|
|
1289
|
+
initially_bound = bound_in_block - bound_before_block
|
|
1290
|
+
|
|
1291
|
+
return BlockBindingSnapshot(
|
|
1292
|
+
bound_in_block=bound_in_block,
|
|
1293
|
+
reassigned_in_block=reassigned_in_block,
|
|
1294
|
+
bound_before_block=bound_before_block,
|
|
1295
|
+
bound_after_block=bound_after_block,
|
|
1296
|
+
initially_bound=initially_bound,
|
|
1297
|
+
)
|
|
1298
|
+
|
|
1299
|
+
def _find_return_variables(
|
|
1300
|
+
self,
|
|
1301
|
+
func: FunctionNode,
|
|
1302
|
+
block_range: Tuple[int, int],
|
|
1303
|
+
initially_bound: Set[str],
|
|
1304
|
+
*,
|
|
1305
|
+
debug_label: Optional[str] = None,
|
|
1306
|
+
) -> Set[str]:
|
|
1307
|
+
"""
|
|
1308
|
+
Determine which newly-bound variables are read after the block (DRY helper parity).
|
|
1309
|
+
"""
|
|
1310
|
+
|
|
1311
|
+
if not initially_bound:
|
|
1312
|
+
return set()
|
|
1313
|
+
|
|
1314
|
+
block_end_line = block_range[1]
|
|
1315
|
+
result: Set[str] = set()
|
|
1316
|
+
debug_enabled = bool(os.getenv("DEBUG_VALIDATION"))
|
|
1317
|
+
|
|
1318
|
+
for stmt in func.body:
|
|
1319
|
+
if not hasattr(stmt, "lineno") or stmt.lineno <= block_end_line:
|
|
1320
|
+
continue
|
|
1321
|
+
|
|
1322
|
+
uses = self._get_used_names(stmt)
|
|
1323
|
+
if debug_enabled and debug_label:
|
|
1324
|
+
print(
|
|
1325
|
+
f" {debug_label}: stmt@{stmt.lineno} ({stmt.__class__.__name__}) uses {uses}"
|
|
1326
|
+
)
|
|
1327
|
+
|
|
1328
|
+
overlap = uses & initially_bound
|
|
1329
|
+
if overlap:
|
|
1330
|
+
result.update(overlap)
|
|
1331
|
+
if debug_enabled and debug_label:
|
|
1332
|
+
print(
|
|
1333
|
+
f" RETURN NEEDED ({debug_label}): Variable(s) {overlap} will be returned from extracted function"
|
|
1334
|
+
)
|
|
1335
|
+
|
|
1336
|
+
if debug_enabled and debug_label and result:
|
|
1337
|
+
print(f"{debug_label} requires returning: {result}")
|
|
1338
|
+
|
|
1339
|
+
return result
|
|
1340
|
+
|
|
1341
|
+
def _find_block_pairs_multi_file(
|
|
1342
|
+
self,
|
|
1343
|
+
all_functions: List[
|
|
1344
|
+
Tuple[
|
|
1345
|
+
str,
|
|
1346
|
+
FunctionNode,
|
|
1347
|
+
str,
|
|
1348
|
+
ScopeAnalyzer,
|
|
1349
|
+
Scope,
|
|
1350
|
+
Optional[str],
|
|
1351
|
+
Optional[str],
|
|
1352
|
+
List[str],
|
|
1353
|
+
]
|
|
1354
|
+
],
|
|
1355
|
+
*,
|
|
1356
|
+
progress: str = "none",
|
|
1357
|
+
) -> List[CodeBlockPair]:
|
|
1358
|
+
"""
|
|
1359
|
+
Find all non-overlapping pairs of code blocks across multiple files.
|
|
1360
|
+
|
|
1361
|
+
Args:
|
|
1362
|
+
all_functions: List of (file_path, function, source, scope_analyzer, root_scope)
|
|
1363
|
+
|
|
1364
|
+
Returns:
|
|
1365
|
+
List of code block pairs
|
|
1366
|
+
"""
|
|
1367
|
+
pairs = []
|
|
1368
|
+
|
|
1369
|
+
# Progress setup
|
|
1370
|
+
use_tqdm = progress in ("tqdm", "auto")
|
|
1371
|
+
tqdm_bar = None
|
|
1372
|
+
total_funcs = len(all_functions)
|
|
1373
|
+
total_func_pairs = (total_funcs * (total_funcs - 1)) // 2 if total_funcs > 1 else 0
|
|
1374
|
+
if use_tqdm and total_func_pairs > 0:
|
|
1375
|
+
tqdm_cls = self._load_tqdm_wrapper()
|
|
1376
|
+
if tqdm_cls is not None:
|
|
1377
|
+
tqdm_bar = tqdm_cls(
|
|
1378
|
+
total=total_func_pairs,
|
|
1379
|
+
desc="pairs",
|
|
1380
|
+
unit="fp",
|
|
1381
|
+
dynamic_ncols=True,
|
|
1382
|
+
leave=False,
|
|
1383
|
+
)
|
|
1384
|
+
else:
|
|
1385
|
+
use_tqdm = False
|
|
1386
|
+
|
|
1387
|
+
use_inline = (not use_tqdm) and progress in ("tqdm", "auto") and total_func_pairs > 0
|
|
1388
|
+
last_pct = -1
|
|
1389
|
+
self._start_inline_status("Pairing blocks:", use_inline)
|
|
1390
|
+
|
|
1391
|
+
func_pairs_done = 0
|
|
1392
|
+
|
|
1393
|
+
# For each pair of functions (including across files)
|
|
1394
|
+
for i, entry1 in enumerate(all_functions):
|
|
1395
|
+
# Backward compatibility: allow 5-tuples (no class context)
|
|
1396
|
+
if len(entry1) >= 8:
|
|
1397
|
+
file1, func1, source1, analyzer1, scope1, class1, encl1, anc1 = entry1
|
|
1398
|
+
elif len(entry1) == 7:
|
|
1399
|
+
file1, func1, source1, analyzer1, scope1, class1, encl1 = entry1
|
|
1400
|
+
anc1 = []
|
|
1401
|
+
else:
|
|
1402
|
+
file1, func1, source1, analyzer1, scope1 = entry1
|
|
1403
|
+
class1 = None
|
|
1404
|
+
encl1 = None
|
|
1405
|
+
anc1 = []
|
|
1406
|
+
|
|
1407
|
+
for entry2 in all_functions[i + 1 :]:
|
|
1408
|
+
if len(entry2) >= 8:
|
|
1409
|
+
file2, func2, source2, analyzer2, scope2, class2, encl2, anc2 = entry2
|
|
1410
|
+
elif len(entry2) == 7:
|
|
1411
|
+
file2, func2, source2, analyzer2, scope2, class2, encl2 = entry2
|
|
1412
|
+
anc2 = []
|
|
1413
|
+
else:
|
|
1414
|
+
file2, func2, source2, analyzer2, scope2 = entry2
|
|
1415
|
+
class2 = None
|
|
1416
|
+
encl2 = None
|
|
1417
|
+
anc2 = []
|
|
1418
|
+
# Extract all code blocks from each function
|
|
1419
|
+
blocks1 = [
|
|
1420
|
+
(block_range, block_nodes, extract_block_signature(block_nodes))
|
|
1421
|
+
for block_range, block_nodes in self._extract_code_blocks(func1)
|
|
1422
|
+
]
|
|
1423
|
+
blocks2 = [
|
|
1424
|
+
(block_range, block_nodes, extract_block_signature(block_nodes))
|
|
1425
|
+
for block_range, block_nodes in self._extract_code_blocks(func2)
|
|
1426
|
+
]
|
|
1427
|
+
|
|
1428
|
+
# Compare all pairs of blocks
|
|
1429
|
+
for block1_range, block1_nodes, sig1 in blocks1:
|
|
1430
|
+
for block2_range, block2_nodes, sig2 in blocks2:
|
|
1431
|
+
# Must have same number of statements for unification
|
|
1432
|
+
if len(block1_nodes) != len(block2_nodes):
|
|
1433
|
+
continue
|
|
1434
|
+
|
|
1435
|
+
# Check minimum size
|
|
1436
|
+
start1, end1 = block1_range
|
|
1437
|
+
start2, end2 = block2_range
|
|
1438
|
+
if (end1 - start1 + 1) < self.min_lines or (
|
|
1439
|
+
end2 - start2 + 1
|
|
1440
|
+
) < self.min_lines:
|
|
1441
|
+
continue
|
|
1442
|
+
|
|
1443
|
+
if not quick_filter(sig1, sig2):
|
|
1444
|
+
continue
|
|
1445
|
+
|
|
1446
|
+
# Create pair with all necessary context
|
|
1447
|
+
pair = CodeBlockPair(
|
|
1448
|
+
file_path=file1,
|
|
1449
|
+
function1_name=func1.name,
|
|
1450
|
+
function2_name=func2.name,
|
|
1451
|
+
block1_range=block1_range,
|
|
1452
|
+
block2_range=block2_range,
|
|
1453
|
+
block1_nodes=block1_nodes,
|
|
1454
|
+
block2_nodes=block2_nodes,
|
|
1455
|
+
file_path2=file2,
|
|
1456
|
+
class1_name=class1,
|
|
1457
|
+
class2_name=class2,
|
|
1458
|
+
enclosing_function1_name=encl1,
|
|
1459
|
+
enclosing_function2_name=encl2,
|
|
1460
|
+
function1_ancestry=anc1,
|
|
1461
|
+
function2_ancestry=anc2,
|
|
1462
|
+
scope_analyzer1=analyzer1,
|
|
1463
|
+
scope_analyzer2=analyzer2,
|
|
1464
|
+
root_scope1=scope1,
|
|
1465
|
+
root_scope2=scope2,
|
|
1466
|
+
source1=source1,
|
|
1467
|
+
source2=source2,
|
|
1468
|
+
function1_node=func1,
|
|
1469
|
+
function2_node=func2,
|
|
1470
|
+
)
|
|
1471
|
+
pairs.append(pair)
|
|
1472
|
+
|
|
1473
|
+
# Update progress per function pair
|
|
1474
|
+
func_pairs_done += 1
|
|
1475
|
+
if tqdm_bar is not None:
|
|
1476
|
+
try:
|
|
1477
|
+
tqdm_bar.update(1)
|
|
1478
|
+
if func_pairs_done % 20 == 0 or func_pairs_done == total_func_pairs:
|
|
1479
|
+
tqdm_bar.set_postfix({"pairs": len(pairs)}, refresh=True)
|
|
1480
|
+
except Exception:
|
|
1481
|
+
pass
|
|
1482
|
+
elif use_inline:
|
|
1483
|
+
pct = int(100 * func_pairs_done / max(total_func_pairs, 1))
|
|
1484
|
+
if pct != last_pct:
|
|
1485
|
+
last_pct = pct
|
|
1486
|
+
self._update_inline_status(
|
|
1487
|
+
"Pairing blocks:",
|
|
1488
|
+
pct,
|
|
1489
|
+
suffix=f"| pairs={len(pairs)}",
|
|
1490
|
+
)
|
|
1491
|
+
if tqdm_bar is not None:
|
|
1492
|
+
try:
|
|
1493
|
+
tqdm_bar.close()
|
|
1494
|
+
except Exception:
|
|
1495
|
+
pass
|
|
1496
|
+
self._finish_inline_status(use_inline)
|
|
1497
|
+
|
|
1498
|
+
return pairs
|
|
1499
|
+
|
|
1500
|
+
def _try_refactor_pair_multi_file(
|
|
1501
|
+
self,
|
|
1502
|
+
pair: CodeBlockPair,
|
|
1503
|
+
all_functions: List[
|
|
1504
|
+
Tuple[
|
|
1505
|
+
str,
|
|
1506
|
+
FunctionNode,
|
|
1507
|
+
str,
|
|
1508
|
+
ScopeAnalyzer,
|
|
1509
|
+
Scope,
|
|
1510
|
+
Optional[str],
|
|
1511
|
+
Optional[str],
|
|
1512
|
+
List[str],
|
|
1513
|
+
]
|
|
1514
|
+
],
|
|
1515
|
+
class_infos: List[ClassInfo],
|
|
1516
|
+
) -> Optional[RefactoringProposal]:
|
|
1517
|
+
"""
|
|
1518
|
+
Try to refactor a pair of code blocks using unification (cross-file support).
|
|
1519
|
+
|
|
1520
|
+
Args:
|
|
1521
|
+
pair: Code block pair (may be cross-file)
|
|
1522
|
+
all_functions: All functions being analyzed
|
|
1523
|
+
class_infos: Metadata about classes discovered in analyzed files
|
|
1524
|
+
|
|
1525
|
+
Returns:
|
|
1526
|
+
Refactoring proposal or None
|
|
1527
|
+
"""
|
|
1528
|
+
# DEBUG logging
|
|
1529
|
+
import os
|
|
1530
|
+
|
|
1531
|
+
if os.getenv("DEBUG_VALIDATION"):
|
|
1532
|
+
print("\n=== _try_refactor_pair_multi_file called ===")
|
|
1533
|
+
print(f"Functions: {pair.function1_name} and {pair.function2_name}")
|
|
1534
|
+
print(f"Block1 range: {pair.block1_range}")
|
|
1535
|
+
print(f"Block2 range: {pair.block2_range}")
|
|
1536
|
+
|
|
1537
|
+
# Resolve contextual analyzers and scopes from the aggregated function list
|
|
1538
|
+
func1: Optional[FunctionNode] = pair.function1_node
|
|
1539
|
+
func2: Optional[FunctionNode] = pair.function2_node
|
|
1540
|
+
scope_analyzer1: Optional[ScopeAnalyzer] = None
|
|
1541
|
+
scope_analyzer2: Optional[ScopeAnalyzer] = None
|
|
1542
|
+
root_scope1: Optional[Scope] = None
|
|
1543
|
+
root_scope2: Optional[Scope] = None
|
|
1544
|
+
|
|
1545
|
+
for entry in all_functions:
|
|
1546
|
+
(
|
|
1547
|
+
file_path,
|
|
1548
|
+
func,
|
|
1549
|
+
_source,
|
|
1550
|
+
analyzer,
|
|
1551
|
+
root_scope_entry,
|
|
1552
|
+
_class_name,
|
|
1553
|
+
_enclosing_func,
|
|
1554
|
+
_ancestry,
|
|
1555
|
+
) = entry
|
|
1556
|
+
if func1 is None and file_path == pair.file_path and func.name == pair.function1_name:
|
|
1557
|
+
func1 = func
|
|
1558
|
+
scope_analyzer1 = analyzer
|
|
1559
|
+
root_scope1 = root_scope_entry
|
|
1560
|
+
if (
|
|
1561
|
+
func2 is None
|
|
1562
|
+
and file_path == (pair.file_path2 or pair.file_path)
|
|
1563
|
+
and func.name == pair.function2_name
|
|
1564
|
+
):
|
|
1565
|
+
func2 = func
|
|
1566
|
+
scope_analyzer2 = analyzer
|
|
1567
|
+
root_scope2 = root_scope_entry
|
|
1568
|
+
if func1 is not None and func2 is not None:
|
|
1569
|
+
break
|
|
1570
|
+
|
|
1571
|
+
# Fallback to the pair-provided analyzers/scopes when discovery fails
|
|
1572
|
+
scope_analyzer = scope_analyzer1 or pair.scope_analyzer1
|
|
1573
|
+
root_scope = root_scope1 or pair.root_scope1
|
|
1574
|
+
if scope_analyzer2 is None and pair.scope_analyzer2 is not None:
|
|
1575
|
+
scope_analyzer2 = pair.scope_analyzer2
|
|
1576
|
+
if root_scope2 is None and pair.root_scope2 is not None:
|
|
1577
|
+
root_scope2 = pair.root_scope2
|
|
1578
|
+
|
|
1579
|
+
method_info1 = self._get_method_context(func1, pair.class1_name)
|
|
1580
|
+
method_info2 = self._get_method_context(func2, pair.class2_name)
|
|
1581
|
+
|
|
1582
|
+
# (Removed specialized full-body extraction fast-path; reverting to generic pairing logic.)
|
|
1583
|
+
|
|
1584
|
+
# DEBUG logging
|
|
1585
|
+
import os
|
|
1586
|
+
|
|
1587
|
+
if os.getenv("DEBUG_VALIDATION"):
|
|
1588
|
+
print("\n=== Finding Functions ===")
|
|
1589
|
+
print(f"Looking for: {pair.function1_name} and {pair.function2_name}")
|
|
1590
|
+
print(f"Found func1: {func1 is not None}")
|
|
1591
|
+
print(f"Found func2: {func2 is not None}")
|
|
1592
|
+
|
|
1593
|
+
# CRITICAL: Validate that blocks don't contain reassignments without initial bindings
|
|
1594
|
+
# Initialize return_variables tracking
|
|
1595
|
+
# This will be populated if we find variables that need to be returned
|
|
1596
|
+
return_variables_block1: Set[str] = set()
|
|
1597
|
+
return_variables_block2: Set[str] = set()
|
|
1598
|
+
|
|
1599
|
+
bound_in_block1: Set[str] = set()
|
|
1600
|
+
bound_before_block1: Set[str] = set()
|
|
1601
|
+
bound_after_block1: Set[str] = set()
|
|
1602
|
+
initially_bound1: Set[str] = set()
|
|
1603
|
+
|
|
1604
|
+
bound_in_block2: Set[str] = set()
|
|
1605
|
+
bound_before_block2: Set[str] = set()
|
|
1606
|
+
bound_after_block2: Set[str] = set()
|
|
1607
|
+
initially_bound2: Set[str] = set()
|
|
1608
|
+
|
|
1609
|
+
# This prevents extracting code like "result = result + 10" when "result = x * 2"
|
|
1610
|
+
# is outside the block. Such extractions are fundamentally unsound.
|
|
1611
|
+
if func1 and func2:
|
|
1612
|
+
# Analyze assignments in both functions
|
|
1613
|
+
reassignments1 = self._get_assignment_reuse(func1)
|
|
1614
|
+
reassignments2 = self._get_assignment_reuse(func2)
|
|
1615
|
+
|
|
1616
|
+
# Check if block1 contains reassignments without bindings
|
|
1617
|
+
has_unsafe1, problematic_vars1 = has_reassignments_without_bindings(
|
|
1618
|
+
func1, pair.block1_nodes, reassignments1
|
|
1619
|
+
)
|
|
1620
|
+
if has_unsafe1:
|
|
1621
|
+
self._debug_reject("unsafe_reassignment_block1", pair, str(problematic_vars1))
|
|
1622
|
+
return None
|
|
1623
|
+
|
|
1624
|
+
# Check if block2 contains reassignments without bindings
|
|
1625
|
+
has_unsafe2, problematic_vars2 = has_reassignments_without_bindings(
|
|
1626
|
+
func2, pair.block2_nodes, reassignments2
|
|
1627
|
+
)
|
|
1628
|
+
if has_unsafe2:
|
|
1629
|
+
self._debug_reject("unsafe_reassignment_block2", pair, str(problematic_vars2))
|
|
1630
|
+
return None
|
|
1631
|
+
|
|
1632
|
+
# Consolidated helper mirrors DRY's _refactor_engine_helper_2 (docs/DRY_RUN_2025-11-28.md)
|
|
1633
|
+
block1_snapshot = self._build_block_binding_snapshot(
|
|
1634
|
+
func1, pair.block1_nodes, pair.block1_range, reassignments1
|
|
1635
|
+
)
|
|
1636
|
+
block2_snapshot = self._build_block_binding_snapshot(
|
|
1637
|
+
func2, pair.block2_nodes, pair.block2_range, reassignments2
|
|
1638
|
+
)
|
|
1639
|
+
|
|
1640
|
+
bound_in_block1 = block1_snapshot.bound_in_block
|
|
1641
|
+
bound_before_block1 = block1_snapshot.bound_before_block
|
|
1642
|
+
bound_after_block1 = block1_snapshot.bound_after_block
|
|
1643
|
+
initially_bound1 = block1_snapshot.initially_bound
|
|
1644
|
+
|
|
1645
|
+
bound_in_block2 = block2_snapshot.bound_in_block
|
|
1646
|
+
bound_before_block2 = block2_snapshot.bound_before_block
|
|
1647
|
+
bound_after_block2 = block2_snapshot.bound_after_block
|
|
1648
|
+
initially_bound2 = block2_snapshot.initially_bound
|
|
1649
|
+
|
|
1650
|
+
debug_enabled = bool(os.getenv("DEBUG_VALIDATION"))
|
|
1651
|
+
if debug_enabled:
|
|
1652
|
+
print("\n=== Block1 Validation Debug ===")
|
|
1653
|
+
print(f"Function: {pair.function1_name}")
|
|
1654
|
+
print(f"Block lines: {pair.block1_range}")
|
|
1655
|
+
print(f"Bound in block: {bound_in_block1}")
|
|
1656
|
+
print(f"Bound before block: {bound_before_block1}")
|
|
1657
|
+
print(f"Newly bound in block: {initially_bound1}")
|
|
1658
|
+
|
|
1659
|
+
return_variables_block1 = self._find_return_variables(
|
|
1660
|
+
func1,
|
|
1661
|
+
pair.block1_range,
|
|
1662
|
+
initially_bound1,
|
|
1663
|
+
debug_label="Block1 Validation Debug" if debug_enabled else None,
|
|
1664
|
+
)
|
|
1665
|
+
|
|
1666
|
+
if debug_enabled:
|
|
1667
|
+
print("\n=== Block2 Validation Debug ===")
|
|
1668
|
+
print(f"Function: {pair.function2_name}")
|
|
1669
|
+
print(f"Block lines: {pair.block2_range}")
|
|
1670
|
+
print(f"Bound in block: {bound_in_block2}")
|
|
1671
|
+
print(f"Bound before block: {bound_before_block2}")
|
|
1672
|
+
print(f"Newly bound in block: {initially_bound2}")
|
|
1673
|
+
|
|
1674
|
+
return_variables_block2 = self._find_return_variables(
|
|
1675
|
+
func2,
|
|
1676
|
+
pair.block2_range,
|
|
1677
|
+
initially_bound2,
|
|
1678
|
+
debug_label="Block2 Validation Debug" if debug_enabled else None,
|
|
1679
|
+
)
|
|
1680
|
+
|
|
1681
|
+
# Check if both blocks are value-producing or both are not
|
|
1682
|
+
# Blocks with return_variables are treated as value-producing because
|
|
1683
|
+
# we will add return statements for those variables
|
|
1684
|
+
value_prod1 = is_value_producing(cast(List[ast.stmt], pair.block1_nodes)) or bool(
|
|
1685
|
+
return_variables_block1
|
|
1686
|
+
)
|
|
1687
|
+
value_prod2 = is_value_producing(cast(List[ast.stmt], pair.block2_nodes)) or bool(
|
|
1688
|
+
return_variables_block2
|
|
1689
|
+
)
|
|
1690
|
+
|
|
1691
|
+
if os.getenv("DEBUG_VALIDATION"):
|
|
1692
|
+
print(f" Value-producing check: block1={value_prod1}, block2={value_prod2}")
|
|
1693
|
+
if return_variables_block1:
|
|
1694
|
+
print(f" Block1 has return_variables: {return_variables_block1}")
|
|
1695
|
+
if return_variables_block2:
|
|
1696
|
+
print(f" Block2 has return_variables: {return_variables_block2}")
|
|
1697
|
+
|
|
1698
|
+
if value_prod1 != value_prod2:
|
|
1699
|
+
if os.getenv("DEBUG_VALIDATION"):
|
|
1700
|
+
print(" REJECTED: Value-producing mismatch")
|
|
1701
|
+
self._debug_reject("value_producing_mismatch", pair)
|
|
1702
|
+
return None
|
|
1703
|
+
|
|
1704
|
+
# CRITICAL: If blocks are NATURALLY value-producing (have return statements),
|
|
1705
|
+
# ensure complete return coverage. This prevents extracting partial control flow.
|
|
1706
|
+
# Skip this check for blocks that will have return statements ADDED for return_variables
|
|
1707
|
+
if value_prod1 and not return_variables_block1: # Naturally value-producing
|
|
1708
|
+
from .extractor import has_complete_return_coverage
|
|
1709
|
+
|
|
1710
|
+
if not has_complete_return_coverage(cast(List[ast.stmt], pair.block1_nodes)):
|
|
1711
|
+
if os.getenv("DEBUG_VALIDATION"):
|
|
1712
|
+
print(" REJECTED: Block1 missing complete return coverage")
|
|
1713
|
+
self._debug_reject("incomplete_return_coverage_block1", pair)
|
|
1714
|
+
return None
|
|
1715
|
+
if not has_complete_return_coverage(cast(List[ast.stmt], pair.block2_nodes)):
|
|
1716
|
+
if os.getenv("DEBUG_VALIDATION"):
|
|
1717
|
+
print(" REJECTED: Block2 missing complete return coverage")
|
|
1718
|
+
self._debug_reject("incomplete_return_coverage_block2", pair)
|
|
1719
|
+
return None
|
|
1720
|
+
|
|
1721
|
+
# Heuristic: avoid extracting trivial single-line return blocks that just
|
|
1722
|
+
# return a previously bound local name (e.g., `return result`). Prefer
|
|
1723
|
+
# extracting the preceding computation that produces the value.
|
|
1724
|
+
def _is_trivial_return_of_bound_name(
|
|
1725
|
+
block_nodes: List[ast.AST], bound_before_block: Set[str], bound_in_block: Set[str]
|
|
1726
|
+
) -> bool:
|
|
1727
|
+
if len(block_nodes) != 1:
|
|
1728
|
+
return False
|
|
1729
|
+
stmt = block_nodes[0]
|
|
1730
|
+
return (
|
|
1731
|
+
isinstance(stmt, ast.Return)
|
|
1732
|
+
and isinstance(stmt.value, ast.Name)
|
|
1733
|
+
and stmt.value.id in bound_before_block
|
|
1734
|
+
and stmt.value.id not in bound_in_block
|
|
1735
|
+
)
|
|
1736
|
+
|
|
1737
|
+
if _is_trivial_return_of_bound_name(
|
|
1738
|
+
pair.block1_nodes, bound_before_block1, bound_in_block1
|
|
1739
|
+
) and _is_trivial_return_of_bound_name(
|
|
1740
|
+
pair.block2_nodes, bound_before_block2, bound_in_block2
|
|
1741
|
+
):
|
|
1742
|
+
if os.getenv("DEBUG_VALIDATION"):
|
|
1743
|
+
print(
|
|
1744
|
+
" REJECTED: Trivial single-line return blocks (prefer extracting computation)"
|
|
1745
|
+
)
|
|
1746
|
+
self._debug_reject("trivial_return_blocks", pair)
|
|
1747
|
+
return None
|
|
1748
|
+
|
|
1749
|
+
# Check structural similarity
|
|
1750
|
+
if not self._are_structurally_similar(pair.block1_nodes, pair.block2_nodes):
|
|
1751
|
+
if os.getenv("DEBUG_VALIDATION"):
|
|
1752
|
+
print(" REJECTED: Not structurally similar")
|
|
1753
|
+
self._debug_reject("not_structurally_similar", pair)
|
|
1754
|
+
return None
|
|
1755
|
+
|
|
1756
|
+
# Attempt unification
|
|
1757
|
+
blocks = [pair.block1_nodes, pair.block2_nodes]
|
|
1758
|
+
hygienic_renames: List[Dict[str, str]] = [{}, {}]
|
|
1759
|
+
|
|
1760
|
+
if os.getenv("DEBUG_VALIDATION"):
|
|
1761
|
+
print(" Attempting unification...")
|
|
1762
|
+
|
|
1763
|
+
try:
|
|
1764
|
+
substitution = self.unifier.unify_blocks(blocks, hygienic_renames)
|
|
1765
|
+
except Exception as e:
|
|
1766
|
+
if os.getenv("DEBUG_VALIDATION"):
|
|
1767
|
+
print(f" REJECTED: Unification exception: {e}")
|
|
1768
|
+
self._debug_reject("unification_exception", pair, str(e))
|
|
1769
|
+
return None
|
|
1770
|
+
|
|
1771
|
+
if not substitution:
|
|
1772
|
+
if os.getenv("DEBUG_VALIDATION"):
|
|
1773
|
+
print(" REJECTED: Unification failed (no substitution)")
|
|
1774
|
+
self._debug_reject("unification_failed", pair)
|
|
1775
|
+
return None
|
|
1776
|
+
|
|
1777
|
+
if os.getenv("DEBUG_VALIDATION"):
|
|
1778
|
+
print(" ✓ Unification successful")
|
|
1779
|
+
print(f" Substitution: {substitution}")
|
|
1780
|
+
|
|
1781
|
+
# Pre-compute the deepest common enclosing function (for same-file cases)
|
|
1782
|
+
dce_insert_func: Optional[str] = None
|
|
1783
|
+
try:
|
|
1784
|
+
same_file_ctx = pair.file_path2 is not None and pair.file_path2 == pair.file_path
|
|
1785
|
+
if same_file_ctx:
|
|
1786
|
+
dce_insert_func = self._deepest_common_ancestry(
|
|
1787
|
+
pair.function1_ancestry, pair.function2_ancestry
|
|
1788
|
+
)
|
|
1789
|
+
except Exception:
|
|
1790
|
+
dce_insert_func = None
|
|
1791
|
+
|
|
1792
|
+
# Get enclosing names to avoid shadowing (module-level by default)
|
|
1793
|
+
enclosing_names = set(root_scope.bindings.keys()) if root_scope else set()
|
|
1794
|
+
# If we plan to insert into a specific function scope (DCE), enrich hygiene set with that
|
|
1795
|
+
# function's local bindings to avoid name collisions
|
|
1796
|
+
if dce_insert_func:
|
|
1797
|
+
try:
|
|
1798
|
+
for fpath, fn, _src, analyzer, rscope, _cls, _encl, _anc in all_functions:
|
|
1799
|
+
if fpath == (pair.file_path2 or pair.file_path) and fn.name == dce_insert_func:
|
|
1800
|
+
func_scope = analyzer.node_scopes.get(fn)
|
|
1801
|
+
if func_scope is not None:
|
|
1802
|
+
enclosing_names.update(func_scope.bindings.keys())
|
|
1803
|
+
break
|
|
1804
|
+
except Exception:
|
|
1805
|
+
# Best effort only
|
|
1806
|
+
pass
|
|
1807
|
+
# Hygiene improvement: if we'll insert into a specific function scope (DCE),
|
|
1808
|
+
# include that function's local bindings to avoid name collisions.
|
|
1809
|
+
try:
|
|
1810
|
+
same_file_for_hygiene = (
|
|
1811
|
+
pair.file_path2 is not None and pair.file_path2 == pair.file_path
|
|
1812
|
+
)
|
|
1813
|
+
|
|
1814
|
+
target_insert_fn: Optional[str] = None
|
|
1815
|
+
if same_file_for_hygiene:
|
|
1816
|
+
target_insert_fn = self._deepest_common_ancestry(
|
|
1817
|
+
pair.function1_ancestry, pair.function2_ancestry
|
|
1818
|
+
)
|
|
1819
|
+
if target_insert_fn:
|
|
1820
|
+
# Locate the target function node and its scope analyzer for this file
|
|
1821
|
+
target_func_node = None
|
|
1822
|
+
target_analyzer: Optional[ScopeAnalyzer] = None
|
|
1823
|
+
for fpath, fn, _src, analyzer, _rscope, _cls, _encl, _anc in all_functions:
|
|
1824
|
+
if fpath == pair.file_path and fn.name == target_insert_fn:
|
|
1825
|
+
target_func_node = fn
|
|
1826
|
+
target_analyzer = analyzer
|
|
1827
|
+
break
|
|
1828
|
+
if target_func_node is not None and target_analyzer is not None:
|
|
1829
|
+
func_scope = target_analyzer.node_scopes.get(target_func_node)
|
|
1830
|
+
if func_scope is not None:
|
|
1831
|
+
enclosing_names.update(func_scope.bindings.keys())
|
|
1832
|
+
except Exception:
|
|
1833
|
+
# Best-effort only; if anything goes wrong, proceed with module-level names
|
|
1834
|
+
pass
|
|
1835
|
+
|
|
1836
|
+
# Compute free variables for both blocks (variables used but not defined in each block)
|
|
1837
|
+
# Use block1's free variables to derive parameters for the extracted function,
|
|
1838
|
+
# but validate incomplete lifetimes independently for each block.
|
|
1839
|
+
free_vars1 = (
|
|
1840
|
+
scope_analyzer.get_free_variables(pair.block1_nodes) if scope_analyzer else set()
|
|
1841
|
+
)
|
|
1842
|
+
free_vars2 = (
|
|
1843
|
+
scope_analyzer2.get_free_variables(pair.block2_nodes) if scope_analyzer2 else set()
|
|
1844
|
+
)
|
|
1845
|
+
|
|
1846
|
+
# CRITICAL VALIDATION: Reject proposals with incomplete variable lifetimes
|
|
1847
|
+
# A free variable bound AFTER the block is problematic - we'd be using it before it's defined.
|
|
1848
|
+
# However, free variables bound BEFORE the block are OK - they become parameters.
|
|
1849
|
+
if free_vars1 & bound_after_block1:
|
|
1850
|
+
incomplete_vars = free_vars1 & bound_after_block1
|
|
1851
|
+
if os.getenv("DEBUG_VALIDATION"):
|
|
1852
|
+
print(
|
|
1853
|
+
f" REJECTED: Block1 uses variables defined AFTER the block: {incomplete_vars}"
|
|
1854
|
+
)
|
|
1855
|
+
print(" These variables would be used before they're defined")
|
|
1856
|
+
self._debug_reject("incomplete_lifetime_block1", pair, str(incomplete_vars))
|
|
1857
|
+
return None
|
|
1858
|
+
|
|
1859
|
+
if free_vars2 & bound_after_block2:
|
|
1860
|
+
incomplete_vars = free_vars2 & bound_after_block2
|
|
1861
|
+
if os.getenv("DEBUG_VALIDATION"):
|
|
1862
|
+
print(
|
|
1863
|
+
f" REJECTED: Block2 uses variables defined AFTER the block: {incomplete_vars}"
|
|
1864
|
+
)
|
|
1865
|
+
self._debug_reject("incomplete_lifetime_block2", pair, str(incomplete_vars))
|
|
1866
|
+
return None
|
|
1867
|
+
|
|
1868
|
+
# Find all variables used in augmented assignments in block1
|
|
1869
|
+
# These variables MUST be passed as parameters even if they appear in substitution
|
|
1870
|
+
# because augmented assignments (total += x) READ the variable before writing it
|
|
1871
|
+
aug_finder = AugAssignFinder()
|
|
1872
|
+
for node in pair.block1_nodes:
|
|
1873
|
+
aug_finder.visit(node)
|
|
1874
|
+
aug_assign_vars = aug_finder.aug_assign_targets
|
|
1875
|
+
|
|
1876
|
+
# Handle augmented assignment variables specially
|
|
1877
|
+
# 1) Remove them from substitution so they remain as free variables/parameters
|
|
1878
|
+
# 2) Record the name mapping per block for call generation later
|
|
1879
|
+
aug_assign_param_mappings: Dict[str, Dict[int, str]] = {}
|
|
1880
|
+
params_to_remove = []
|
|
1881
|
+
for param_name, exprs in list(substitution.param_expressions.items()):
|
|
1882
|
+
for block_idx, expr in exprs:
|
|
1883
|
+
if block_idx == 0 and isinstance(expr, ast.Name) and expr.id in aug_assign_vars:
|
|
1884
|
+
if param_name not in aug_assign_param_mappings:
|
|
1885
|
+
aug_assign_param_mappings[param_name] = {}
|
|
1886
|
+
aug_assign_param_mappings[param_name][block_idx] = expr.id
|
|
1887
|
+
if 0 in aug_assign_param_mappings.get(param_name, {}):
|
|
1888
|
+
params_to_remove.append(param_name)
|
|
1889
|
+
|
|
1890
|
+
# Store the mappings in the substitution object for later use (dynamic attribute)
|
|
1891
|
+
if not hasattr(substitution, "aug_assign_mappings"):
|
|
1892
|
+
setattr(substitution, "aug_assign_mappings", {})
|
|
1893
|
+
aug_mappings = cast(Dict[str, Dict[int, str]], getattr(substitution, "aug_assign_mappings"))
|
|
1894
|
+
for param_name, block_mappings in aug_assign_param_mappings.items():
|
|
1895
|
+
if 0 in block_mappings:
|
|
1896
|
+
block1_var_name = block_mappings[0]
|
|
1897
|
+
aug_mappings[block1_var_name] = block_mappings
|
|
1898
|
+
|
|
1899
|
+
# Remove the augmented assignment parameters from substitution
|
|
1900
|
+
for param_name in params_to_remove:
|
|
1901
|
+
del substitution.param_expressions[param_name]
|
|
1902
|
+
|
|
1903
|
+
# Never parameterize entire f-strings: if any parameter maps to a JoinedStr in
|
|
1904
|
+
# the template block (block 0), remove it so the f-string structure is preserved
|
|
1905
|
+
fstring_params = []
|
|
1906
|
+
for param_name, exprs in substitution.param_expressions.items():
|
|
1907
|
+
for block_idx, expr in exprs:
|
|
1908
|
+
if block_idx == 0 and isinstance(expr, ast.JoinedStr):
|
|
1909
|
+
fstring_params.append(param_name)
|
|
1910
|
+
break
|
|
1911
|
+
for param_name in fstring_params:
|
|
1912
|
+
del substitution.param_expressions[param_name]
|
|
1913
|
+
|
|
1914
|
+
# Remove variables that have been parameterized from free_vars
|
|
1915
|
+
# If a variable was parameterized (e.g., 'user' -> '__param_5'),
|
|
1916
|
+
# it's no longer free - it's been replaced by a parameter
|
|
1917
|
+
# EXCEPT: variables in augmented assignments MUST remain free variables
|
|
1918
|
+
parameterized_vars = set()
|
|
1919
|
+
for param_name, exprs in substitution.param_expressions.items():
|
|
1920
|
+
for block_idx, expr in exprs:
|
|
1921
|
+
# Only check the first block (template block)
|
|
1922
|
+
if block_idx == 0 and isinstance(expr, ast.Name):
|
|
1923
|
+
# Don't add to parameterized_vars if it's an augmented assignment target
|
|
1924
|
+
if expr.id not in aug_assign_vars:
|
|
1925
|
+
parameterized_vars.add(expr.id)
|
|
1926
|
+
|
|
1927
|
+
# Initialize working free_vars from block1's perspective (template block)
|
|
1928
|
+
free_vars = set(free_vars1) - parameterized_vars
|
|
1929
|
+
|
|
1930
|
+
# CRITICAL: Check if any free variables are declared global or nonlocal
|
|
1931
|
+
# If a free variable is global/nonlocal, we cannot parameterize it
|
|
1932
|
+
# because you cannot have a parameter that is also declared global/nonlocal
|
|
1933
|
+
# This would create: SyntaxError: name 'x' is parameter and global
|
|
1934
|
+
assert scope_analyzer is not None
|
|
1935
|
+
func1_scope_id = None
|
|
1936
|
+
for node, scope in scope_analyzer.node_scopes.items():
|
|
1937
|
+
if isinstance(node, ast.FunctionDef) and node.name == pair.function1_name:
|
|
1938
|
+
func1_scope_id = scope.scope_id
|
|
1939
|
+
break
|
|
1940
|
+
|
|
1941
|
+
globals_to_declare_in_extracted: Set[str] = set()
|
|
1942
|
+
nonlocals_to_declare_in_extracted: Set[str] = set()
|
|
1943
|
+
|
|
1944
|
+
if func1_scope_id is not None:
|
|
1945
|
+
# Check if any variables relevant to this block are global or nonlocal in this scope
|
|
1946
|
+
global_vars = scope_analyzer.global_vars.get(func1_scope_id, set())
|
|
1947
|
+
nonlocal_vars = scope_analyzer.nonlocal_vars.get(func1_scope_id, set())
|
|
1948
|
+
|
|
1949
|
+
# For parameterization safety: if a free variable is declared global/nonlocal, do not parameterize it
|
|
1950
|
+
problematic = free_vars & (global_vars | nonlocal_vars)
|
|
1951
|
+
|
|
1952
|
+
# Collect assignment targets and explicit decls inside the template blocks
|
|
1953
|
+
v = AssignTargetVisitor()
|
|
1954
|
+
for n in pair.block1_nodes:
|
|
1955
|
+
v.visit(n)
|
|
1956
|
+
for n in pair.block2_nodes:
|
|
1957
|
+
v.visit(n)
|
|
1958
|
+
assigned_names = v.assigned_names
|
|
1959
|
+
declared_global_in_block = v.declared_global_in_block
|
|
1960
|
+
declared_nonlocal_in_block = v.declared_nonlocal_in_block
|
|
1961
|
+
|
|
1962
|
+
# Names that are assigned within the block and are global/nonlocal in the enclosing function
|
|
1963
|
+
# MUST be declared in the extracted helper to preserve assignment semantics,
|
|
1964
|
+
# even if they are not free variables of the original block.
|
|
1965
|
+
assigned_problematic_any = assigned_names & (global_vars | nonlocal_vars)
|
|
1966
|
+
|
|
1967
|
+
# For assigned globals/nonlocals that weren't explicitly declared within the block,
|
|
1968
|
+
# promote the declaration into the extracted function body.
|
|
1969
|
+
globals_to_declare_in_extracted = (
|
|
1970
|
+
assigned_problematic_any & global_vars
|
|
1971
|
+
) - declared_global_in_block
|
|
1972
|
+
nonlocals_to_declare_in_extracted = (
|
|
1973
|
+
assigned_problematic_any & nonlocal_vars
|
|
1974
|
+
) - declared_nonlocal_in_block
|
|
1975
|
+
|
|
1976
|
+
# Do not parameterize free variables that are global/nonlocal; let them remain free
|
|
1977
|
+
# so the extracted function references the outer binding.
|
|
1978
|
+
free_vars -= problematic
|
|
1979
|
+
|
|
1980
|
+
# Extract function
|
|
1981
|
+
try:
|
|
1982
|
+
func_def, param_order = self.extractor.extract_function(
|
|
1983
|
+
template_block=pair.block1_nodes,
|
|
1984
|
+
substitution=substitution,
|
|
1985
|
+
free_variables=free_vars,
|
|
1986
|
+
enclosing_names=enclosing_names,
|
|
1987
|
+
is_value_producing=value_prod1,
|
|
1988
|
+
return_variables=list(return_variables_block1),
|
|
1989
|
+
global_decls=(
|
|
1990
|
+
globals_to_declare_in_extracted if globals_to_declare_in_extracted else None
|
|
1991
|
+
),
|
|
1992
|
+
nonlocal_decls=(
|
|
1993
|
+
nonlocals_to_declare_in_extracted if nonlocals_to_declare_in_extracted else None
|
|
1994
|
+
),
|
|
1995
|
+
# Use hygienic double-underscore name; engine will prefix underscore for methods.
|
|
1996
|
+
function_name="__extracted_func",
|
|
1997
|
+
)
|
|
1998
|
+
except Exception:
|
|
1999
|
+
return None
|
|
2000
|
+
|
|
2001
|
+
# Check for orphaned variables before proceeding
|
|
2002
|
+
# Need to find the function nodes to check for orphans
|
|
2003
|
+
func1_for_orphans: Optional[FunctionNode] = None
|
|
2004
|
+
func2_for_orphans: Optional[FunctionNode] = None
|
|
2005
|
+
for entry in all_functions:
|
|
2006
|
+
file_path, func = entry[0], entry[1]
|
|
2007
|
+
if file_path == pair.file_path and func.name == pair.function1_name:
|
|
2008
|
+
func1_for_orphans = func
|
|
2009
|
+
if (
|
|
2010
|
+
file_path == (pair.file_path2 or pair.file_path)
|
|
2011
|
+
and func.name == pair.function2_name
|
|
2012
|
+
):
|
|
2013
|
+
func2_for_orphans = func
|
|
2014
|
+
|
|
2015
|
+
if func1_for_orphans and func2_for_orphans:
|
|
2016
|
+
# Get block indices within their respective function bodies
|
|
2017
|
+
indices1 = self._get_block_indices(func1_for_orphans, pair.block1_nodes)
|
|
2018
|
+
indices2 = self._get_block_indices(func2_for_orphans, pair.block2_nodes)
|
|
2019
|
+
|
|
2020
|
+
if indices1 and indices2:
|
|
2021
|
+
# Get function bodies (skip docstring)
|
|
2022
|
+
body1 = func1_for_orphans.body
|
|
2023
|
+
start_idx1 = 0
|
|
2024
|
+
if (
|
|
2025
|
+
body1
|
|
2026
|
+
and isinstance(body1[0], ast.Expr)
|
|
2027
|
+
and isinstance(body1[0].value, ast.Constant)
|
|
2028
|
+
and isinstance(body1[0].value.value, str)
|
|
2029
|
+
):
|
|
2030
|
+
start_idx1 = 1
|
|
2031
|
+
body1 = body1[start_idx1:]
|
|
2032
|
+
|
|
2033
|
+
body2 = func2_for_orphans.body
|
|
2034
|
+
start_idx2 = 0
|
|
2035
|
+
if (
|
|
2036
|
+
body2
|
|
2037
|
+
and isinstance(body2[0], ast.Expr)
|
|
2038
|
+
and isinstance(body2[0].value, ast.Constant)
|
|
2039
|
+
and isinstance(body2[0].value.value, str)
|
|
2040
|
+
):
|
|
2041
|
+
start_idx2 = 1
|
|
2042
|
+
body2 = body2[start_idx2:]
|
|
2043
|
+
|
|
2044
|
+
# Check for orphaned variables in both blocks
|
|
2045
|
+
has_orphans1, orphans1 = has_orphaned_variables(
|
|
2046
|
+
cast(List[ast.AST], body1), indices1
|
|
2047
|
+
)
|
|
2048
|
+
has_orphans2, orphans2 = has_orphaned_variables(
|
|
2049
|
+
cast(List[ast.AST], body2), indices2
|
|
2050
|
+
)
|
|
2051
|
+
|
|
2052
|
+
if has_orphans1 or has_orphans2:
|
|
2053
|
+
# Cannot extract - would create orphaned variable references
|
|
2054
|
+
self._debug_reject("orphaned_variables", pair)
|
|
2055
|
+
return None
|
|
2056
|
+
|
|
2057
|
+
# Generate replacement calls
|
|
2058
|
+
replacements: List[Replacement] = []
|
|
2059
|
+
|
|
2060
|
+
# Map block indices to their return variables
|
|
2061
|
+
return_vars_by_block = {0: list(return_variables_block1), 1: list(return_variables_block2)}
|
|
2062
|
+
|
|
2063
|
+
for block_idx, (block_range, file_path) in enumerate(
|
|
2064
|
+
[
|
|
2065
|
+
(pair.block1_range, pair.file_path),
|
|
2066
|
+
(pair.block2_range, pair.file_path2 or pair.file_path),
|
|
2067
|
+
]
|
|
2068
|
+
):
|
|
2069
|
+
try:
|
|
2070
|
+
call_node = self.extractor.generate_call(
|
|
2071
|
+
function_name=func_def.name,
|
|
2072
|
+
block_idx=block_idx,
|
|
2073
|
+
substitution=substitution,
|
|
2074
|
+
param_order=param_order,
|
|
2075
|
+
free_variables=free_vars,
|
|
2076
|
+
is_value_producing=value_prod1,
|
|
2077
|
+
return_variables=return_vars_by_block[block_idx],
|
|
2078
|
+
hygienic_renames=hygienic_renames,
|
|
2079
|
+
)
|
|
2080
|
+
# Guard-rail: validate that the generated call does not reference
|
|
2081
|
+
# undefined names at the call site. This rejects brittle proposals
|
|
2082
|
+
# that leak placeholders (e.g., "__param_1") or invented locals
|
|
2083
|
+
# like "filtered"/"mapped" that are not bound before the block.
|
|
2084
|
+
# Allowed names = variables bound before the block ∪ free variables
|
|
2085
|
+
# (builtins are implicitly allowed by runtime).
|
|
2086
|
+
allowed_before: Set[str]
|
|
2087
|
+
if block_idx == 0:
|
|
2088
|
+
allowed_before = set(bound_before_block1) | set(free_vars1)
|
|
2089
|
+
else:
|
|
2090
|
+
allowed_before = set(bound_before_block2) | set(free_vars2)
|
|
2091
|
+
|
|
2092
|
+
used_in_call = self._get_used_names(call_node)
|
|
2093
|
+
# Whitelist a small set of common builtins used in arguments
|
|
2094
|
+
builtin_whitelist = {
|
|
2095
|
+
"len",
|
|
2096
|
+
"sum",
|
|
2097
|
+
"min",
|
|
2098
|
+
"max",
|
|
2099
|
+
"any",
|
|
2100
|
+
"all",
|
|
2101
|
+
"map",
|
|
2102
|
+
"filter",
|
|
2103
|
+
"sorted",
|
|
2104
|
+
"list",
|
|
2105
|
+
"dict",
|
|
2106
|
+
"set",
|
|
2107
|
+
"range",
|
|
2108
|
+
"int",
|
|
2109
|
+
"float",
|
|
2110
|
+
"str",
|
|
2111
|
+
"bool",
|
|
2112
|
+
"enumerate",
|
|
2113
|
+
"zip",
|
|
2114
|
+
}
|
|
2115
|
+
invalid_names = set()
|
|
2116
|
+
for name in used_in_call:
|
|
2117
|
+
if name == func_def.name:
|
|
2118
|
+
continue # referring to the helper itself is handled elsewhere
|
|
2119
|
+
if name.startswith("__param_"):
|
|
2120
|
+
invalid_names.add(name)
|
|
2121
|
+
continue
|
|
2122
|
+
if name in allowed_before or name in builtin_whitelist:
|
|
2123
|
+
continue
|
|
2124
|
+
invalid_names.add(name)
|
|
2125
|
+
|
|
2126
|
+
if invalid_names:
|
|
2127
|
+
# Reject this proposal as it would introduce undefined names
|
|
2128
|
+
self._debug_reject(
|
|
2129
|
+
"undefined_names_in_call",
|
|
2130
|
+
pair,
|
|
2131
|
+
detail=f"block{block_idx+1}: {sorted(invalid_names)}",
|
|
2132
|
+
)
|
|
2133
|
+
return None
|
|
2134
|
+
# Store file_path and class context
|
|
2135
|
+
class_name = pair.class1_name if block_idx == 0 else pair.class2_name
|
|
2136
|
+
method_info = method_info1 if block_idx == 0 else method_info2
|
|
2137
|
+
replacements.append(
|
|
2138
|
+
Replacement(
|
|
2139
|
+
line_range=block_range,
|
|
2140
|
+
node=call_node,
|
|
2141
|
+
file_path=file_path,
|
|
2142
|
+
class_name=class_name,
|
|
2143
|
+
method_kind=method_info.kind,
|
|
2144
|
+
implicit_param=method_info.implicit_param,
|
|
2145
|
+
)
|
|
2146
|
+
)
|
|
2147
|
+
except Exception:
|
|
2148
|
+
return None
|
|
2149
|
+
|
|
2150
|
+
# Multi-occurrence clustering (same-file): look for additional identical blocks
|
|
2151
|
+
# beyond the initial pair and include them in this proposal as extra replacements.
|
|
2152
|
+
# This helps cases like example1_simple where three functions share the same
|
|
2153
|
+
# validator block; by default pairwise selection would only cover two.
|
|
2154
|
+
try:
|
|
2155
|
+
from .block_signature import extract_block_signature, quick_filter as _qf
|
|
2156
|
+
|
|
2157
|
+
# Only attempt simple clustering for module-level, non-returning validators
|
|
2158
|
+
same_file_ctx = (pair.file_path2 is None) or (pair.file_path2 == pair.file_path)
|
|
2159
|
+
if same_file_ctx and not return_variables_block1 and not return_variables_block2:
|
|
2160
|
+
# Build a set of already covered ranges to avoid duplicates
|
|
2161
|
+
covered = {
|
|
2162
|
+
(pair.file_path, pair.block1_range),
|
|
2163
|
+
(pair.file_path2 or pair.file_path, pair.block2_range),
|
|
2164
|
+
}
|
|
2165
|
+
# Template signature from block1
|
|
2166
|
+
tmpl_sig = extract_block_signature(pair.block1_nodes)
|
|
2167
|
+
|
|
2168
|
+
# Gather candidates from same file functions
|
|
2169
|
+
for entry in all_functions:
|
|
2170
|
+
fpath, fn, _src, analyzerX, _rscopeX, clsX, _enclX, _ancX = (
|
|
2171
|
+
entry if len(entry) >= 8 else (*entry, None, None, None)
|
|
2172
|
+
)
|
|
2173
|
+
if fpath != pair.file_path:
|
|
2174
|
+
continue
|
|
2175
|
+
# Skip the original two functions
|
|
2176
|
+
if fn.name in (pair.function1_name, pair.function2_name):
|
|
2177
|
+
# Still scan, but avoid ranges we've already taken
|
|
2178
|
+
pass
|
|
2179
|
+
# Extract blocks and test quick filter against template
|
|
2180
|
+
for cand_range, cand_nodes in self._extract_code_blocks(fn):
|
|
2181
|
+
if (fpath, cand_range) in covered:
|
|
2182
|
+
continue
|
|
2183
|
+
# Minimum size gate
|
|
2184
|
+
start_line, end_line = cand_range
|
|
2185
|
+
if (end_line - start_line + 1) < self.min_lines:
|
|
2186
|
+
continue
|
|
2187
|
+
cand_sig = extract_block_signature(cand_nodes)
|
|
2188
|
+
if not _qf(tmpl_sig, cand_sig):
|
|
2189
|
+
continue
|
|
2190
|
+
# Try to unify template block with candidate
|
|
2191
|
+
try:
|
|
2192
|
+
subst2 = self.unifier.unify_blocks(
|
|
2193
|
+
[pair.block1_nodes, cand_nodes], [{}, {}]
|
|
2194
|
+
)
|
|
2195
|
+
except Exception:
|
|
2196
|
+
continue
|
|
2197
|
+
if not subst2:
|
|
2198
|
+
continue
|
|
2199
|
+
# Orphan check for candidate within its function body
|
|
2200
|
+
indices = self._get_block_indices(fn, cand_nodes)
|
|
2201
|
+
if indices is None:
|
|
2202
|
+
continue
|
|
2203
|
+
# Skip docstring in body
|
|
2204
|
+
body = self._body_without_docstring(fn.body)
|
|
2205
|
+
has_orph, _orph = has_orphaned_variables(cast(List[ast.AST], body), indices)
|
|
2206
|
+
if has_orph:
|
|
2207
|
+
continue
|
|
2208
|
+
# Generate a call node for the candidate
|
|
2209
|
+
try:
|
|
2210
|
+
call_node2 = self.extractor.generate_call(
|
|
2211
|
+
function_name=func_def.name,
|
|
2212
|
+
block_idx=1,
|
|
2213
|
+
substitution=subst2,
|
|
2214
|
+
param_order=param_order,
|
|
2215
|
+
free_variables=free_vars,
|
|
2216
|
+
is_value_producing=value_prod1,
|
|
2217
|
+
return_variables=[],
|
|
2218
|
+
hygienic_renames=[{}, {}],
|
|
2219
|
+
)
|
|
2220
|
+
except Exception:
|
|
2221
|
+
continue
|
|
2222
|
+
# Validate candidate call-site does not reference undefined names
|
|
2223
|
+
try:
|
|
2224
|
+
used2 = self._get_used_names(call_node2)
|
|
2225
|
+
if any(n.startswith("__param_") for n in used2):
|
|
2226
|
+
# Skip brittle candidate that leaked placeholders
|
|
2227
|
+
continue
|
|
2228
|
+
# Compute names bound before candidate block and free vars within it
|
|
2229
|
+
try:
|
|
2230
|
+
from .assignment_analyzer import (
|
|
2231
|
+
_collect_bindings_and_reassignments as _cbar,
|
|
2232
|
+
)
|
|
2233
|
+
except Exception:
|
|
2234
|
+
_cbar = None # type: ignore
|
|
2235
|
+
|
|
2236
|
+
bound_before_cand: Set[str] = set()
|
|
2237
|
+
if _cbar is not None:
|
|
2238
|
+
reassignX = self._get_assignment_reuse(fn)
|
|
2239
|
+
start_line_cand = cand_range[0]
|
|
2240
|
+
for stmt in fn.body:
|
|
2241
|
+
if hasattr(stmt, "lineno") and stmt.lineno < start_line_cand:
|
|
2242
|
+
bset: Set[str] = set()
|
|
2243
|
+
rset: Set[str] = set()
|
|
2244
|
+
_cbar(stmt, reassignX, bset, rset)
|
|
2245
|
+
bound_before_cand.update(bset)
|
|
2246
|
+
# Treat function params as bound
|
|
2247
|
+
if isinstance(fn, ast.FunctionDef):
|
|
2248
|
+
param_namesX: Set[str] = set()
|
|
2249
|
+
for arg in fn.args.args:
|
|
2250
|
+
param_namesX.add(arg.arg)
|
|
2251
|
+
for arg in getattr(fn.args, "posonlyargs", []) or []:
|
|
2252
|
+
param_namesX.add(arg.arg)
|
|
2253
|
+
for arg in fn.args.kwonlyargs:
|
|
2254
|
+
param_namesX.add(arg.arg)
|
|
2255
|
+
if fn.args.vararg:
|
|
2256
|
+
param_namesX.add(fn.args.vararg.arg)
|
|
2257
|
+
if fn.args.kwarg:
|
|
2258
|
+
param_namesX.add(fn.args.kwarg.arg)
|
|
2259
|
+
bound_before_cand.update(param_namesX)
|
|
2260
|
+
free_vars_cand: Set[str] = set()
|
|
2261
|
+
if analyzerX is not None:
|
|
2262
|
+
try:
|
|
2263
|
+
free_vars_cand = set(analyzerX.get_free_variables(cand_nodes))
|
|
2264
|
+
except Exception:
|
|
2265
|
+
free_vars_cand = set()
|
|
2266
|
+
allowed_cand = bound_before_cand | free_vars_cand
|
|
2267
|
+
builtin_whitelist = {
|
|
2268
|
+
"len",
|
|
2269
|
+
"sum",
|
|
2270
|
+
"min",
|
|
2271
|
+
"max",
|
|
2272
|
+
"any",
|
|
2273
|
+
"all",
|
|
2274
|
+
"map",
|
|
2275
|
+
"filter",
|
|
2276
|
+
"sorted",
|
|
2277
|
+
"list",
|
|
2278
|
+
"dict",
|
|
2279
|
+
"set",
|
|
2280
|
+
"range",
|
|
2281
|
+
"int",
|
|
2282
|
+
"float",
|
|
2283
|
+
"str",
|
|
2284
|
+
"bool",
|
|
2285
|
+
"enumerate",
|
|
2286
|
+
"zip",
|
|
2287
|
+
}
|
|
2288
|
+
invalid2 = set()
|
|
2289
|
+
for name in used2:
|
|
2290
|
+
if name == func_def.name:
|
|
2291
|
+
continue
|
|
2292
|
+
if name in allowed_cand or name in builtin_whitelist:
|
|
2293
|
+
continue
|
|
2294
|
+
invalid2.add(name)
|
|
2295
|
+
if invalid2:
|
|
2296
|
+
# Skip this candidate replacement
|
|
2297
|
+
continue
|
|
2298
|
+
except Exception:
|
|
2299
|
+
# On any validator error, be conservative and skip candidate
|
|
2300
|
+
continue
|
|
2301
|
+
# Append replacement
|
|
2302
|
+
replacements.append(
|
|
2303
|
+
Replacement(
|
|
2304
|
+
line_range=cand_range,
|
|
2305
|
+
node=call_node2,
|
|
2306
|
+
file_path=fpath,
|
|
2307
|
+
class_name=clsX,
|
|
2308
|
+
method_kind=None,
|
|
2309
|
+
implicit_param=None,
|
|
2310
|
+
)
|
|
2311
|
+
)
|
|
2312
|
+
covered.add((fpath, cand_range))
|
|
2313
|
+
except Exception:
|
|
2314
|
+
# Non-fatal: clustering is best-effort only
|
|
2315
|
+
pass
|
|
2316
|
+
# Determine canonical file for extracted function
|
|
2317
|
+
# Default to the first file, but this may change if we insert into an ancestor class
|
|
2318
|
+
canonical_file = pair.file_path
|
|
2319
|
+
|
|
2320
|
+
# Create proposal
|
|
2321
|
+
is_cross_file = pair.file_path2 is not None and pair.file_path != pair.file_path2
|
|
2322
|
+
|
|
2323
|
+
desc = f"Extract common code from {pair.function1_name}"
|
|
2324
|
+
if is_cross_file:
|
|
2325
|
+
assert pair.file_path2 is not None
|
|
2326
|
+
desc += f" ({Path(pair.file_path).name}) and {pair.function2_name} ({Path(pair.file_path2).name})"
|
|
2327
|
+
else:
|
|
2328
|
+
desc += f" and {pair.function2_name}"
|
|
2329
|
+
|
|
2330
|
+
# Default to module-level insertion; when possible insert into deepest common enclosing function.
|
|
2331
|
+
insert_into_class = None
|
|
2332
|
+
insert_into_function = None
|
|
2333
|
+
method_kind_metadata: Optional[Literal["instance", "classmethod", "staticmethod"]] = None
|
|
2334
|
+
method_param_name: Optional[str] = None
|
|
2335
|
+
|
|
2336
|
+
# Consider pairs from the same file even if file_path2 is None (same-file pairing)
|
|
2337
|
+
same_file = (pair.file_path2 is None) or (pair.file_path2 == pair.file_path)
|
|
2338
|
+
|
|
2339
|
+
# Prefer insertion into the function that actually contains BOTH blocks when they come from the
|
|
2340
|
+
# same function (process_user_data & process_admin_data are top-level siblings: no shared ancestry).
|
|
2341
|
+
# If the two functions are the SAME name (duplicates), insert into that function instead of module.
|
|
2342
|
+
if same_file:
|
|
2343
|
+
# Only consider deepest common enclosing function based on ancestry; do NOT use
|
|
2344
|
+
# simple name equality as methods across different classes may share the same
|
|
2345
|
+
# name but are not in the same function scope. Name-equality caused incorrectly
|
|
2346
|
+
# inserting helpers inside one sibling method.
|
|
2347
|
+
if pair.function1_ancestry is not None and pair.function2_ancestry is not None:
|
|
2348
|
+
dce = self._deepest_common_ancestry(
|
|
2349
|
+
pair.function1_ancestry, pair.function2_ancestry
|
|
2350
|
+
)
|
|
2351
|
+
if dce:
|
|
2352
|
+
insert_into_function = dce
|
|
2353
|
+
|
|
2354
|
+
class_plan: Optional[ClassInsertionPlan] = None
|
|
2355
|
+
if insert_into_function is None:
|
|
2356
|
+
class_plan = self._choose_class_insertion(
|
|
2357
|
+
pair,
|
|
2358
|
+
method_info1,
|
|
2359
|
+
method_info2,
|
|
2360
|
+
class_infos,
|
|
2361
|
+
)
|
|
2362
|
+
|
|
2363
|
+
if class_plan is not None:
|
|
2364
|
+
# Refined insertion policy:
|
|
2365
|
+
# 1. If both blocks are from the SAME class -> insert into that class.
|
|
2366
|
+
# 2. If blocks are from DIFFERENT classes and class_plan points to a COMMON ANCESTOR
|
|
2367
|
+
# that is neither of the concrete classes, insert into ancestor (shared visibility).
|
|
2368
|
+
# 3. If class_plan resolves to one of the concrete classes while the other differs ->
|
|
2369
|
+
# fallback to module-level to preserve accessibility (avoid privileging one sibling).
|
|
2370
|
+
same_class = pair.class1_name is not None and pair.class1_name == pair.class2_name
|
|
2371
|
+
target_is_concrete_sibling = (
|
|
2372
|
+
class_plan.class_name in {pair.class1_name, pair.class2_name} and not same_class
|
|
2373
|
+
)
|
|
2374
|
+
if target_is_concrete_sibling:
|
|
2375
|
+
# Helper would become invisible to the other sibling; abort class insertion.
|
|
2376
|
+
class_plan = None
|
|
2377
|
+
else:
|
|
2378
|
+
insert_into_class = class_plan.class_name
|
|
2379
|
+
canonical_file = class_plan.file_path
|
|
2380
|
+
method_kind_metadata = class_plan.method_kind
|
|
2381
|
+
method_param_name = class_plan.implicit_param
|
|
2382
|
+
|
|
2383
|
+
# SAFETY: Avoid refactoring across closures with nonlocal variables for now.
|
|
2384
|
+
# If the containing functions (func1/func2) declare any nonlocal variables, skip this proposal
|
|
2385
|
+
# to preserve known semantics and baseline expectations (e.g., closure_adversarial.py).
|
|
2386
|
+
try:
|
|
2387
|
+
nonlocal_in_func1 = False
|
|
2388
|
+
nonlocal_in_func2 = False
|
|
2389
|
+
if scope_analyzer1 and func1 is not None:
|
|
2390
|
+
scope_id1 = scope_analyzer1.node_scopes.get(func1)
|
|
2391
|
+
if scope_id1:
|
|
2392
|
+
nlv1 = scope_analyzer1.nonlocal_vars.get(scope_id1.scope_id, set())
|
|
2393
|
+
nonlocal_in_func1 = bool(nlv1)
|
|
2394
|
+
elif func1 is not None:
|
|
2395
|
+
nonlocal_in_func1 = self._function_contains_nonlocal(func1)
|
|
2396
|
+
|
|
2397
|
+
if scope_analyzer2 and func2 is not None:
|
|
2398
|
+
scope_id2 = scope_analyzer2.node_scopes.get(func2)
|
|
2399
|
+
if scope_id2:
|
|
2400
|
+
nlv2 = scope_analyzer2.nonlocal_vars.get(scope_id2.scope_id, set())
|
|
2401
|
+
nonlocal_in_func2 = bool(nlv2)
|
|
2402
|
+
elif func2 is not None:
|
|
2403
|
+
nonlocal_in_func2 = self._function_contains_nonlocal(func2)
|
|
2404
|
+
|
|
2405
|
+
if nonlocal_in_func1 or nonlocal_in_func2:
|
|
2406
|
+
self._debug_reject("nonlocal_safety_skip", pair)
|
|
2407
|
+
return None
|
|
2408
|
+
except Exception:
|
|
2409
|
+
# On any analyzer lookup issue, be conservative and proceed without special handling
|
|
2410
|
+
pass
|
|
2411
|
+
|
|
2412
|
+
proposal = RefactoringProposal(
|
|
2413
|
+
file_path=canonical_file,
|
|
2414
|
+
extracted_function=func_def,
|
|
2415
|
+
replacements=replacements, # Now includes file_path
|
|
2416
|
+
description=desc,
|
|
2417
|
+
parameters_count=len(substitution.param_expressions),
|
|
2418
|
+
return_variables=list(return_variables_block1),
|
|
2419
|
+
insert_into_class=insert_into_class,
|
|
2420
|
+
insert_into_function=insert_into_function,
|
|
2421
|
+
method_kind=method_kind_metadata,
|
|
2422
|
+
method_param_name=method_param_name,
|
|
2423
|
+
)
|
|
2424
|
+
|
|
2425
|
+
return proposal
|
|
2426
|
+
|
|
2427
|
+
def apply_refactoring(self, file_path: str, proposal: RefactoringProposal) -> str:
|
|
2428
|
+
"""
|
|
2429
|
+
Apply a refactoring proposal to a file.
|
|
2430
|
+
|
|
2431
|
+
Args:
|
|
2432
|
+
file_path: Path to file
|
|
2433
|
+
proposal: Refactoring proposal
|
|
2434
|
+
|
|
2435
|
+
Returns:
|
|
2436
|
+
Modified source code
|
|
2437
|
+
"""
|
|
2438
|
+
# All proposals now use multi-file format
|
|
2439
|
+
modified_files = self.apply_refactoring_multi_file(proposal)
|
|
2440
|
+
# Backward-compat: handle tuple return (modified_files, changed_paths)
|
|
2441
|
+
if isinstance(modified_files, tuple):
|
|
2442
|
+
modified_files = modified_files[0]
|
|
2443
|
+
# Return the content for the requested file
|
|
2444
|
+
return modified_files.get(file_path, "")
|
|
2445
|
+
|
|
2446
|
+
def apply_refactoring_multi_file(self, proposal: RefactoringProposal) -> Dict[str, str]:
|
|
2447
|
+
"""
|
|
2448
|
+
Apply a cross-file refactoring proposal.
|
|
2449
|
+
|
|
2450
|
+
Args:
|
|
2451
|
+
proposal: Refactoring proposal
|
|
2452
|
+
|
|
2453
|
+
Returns:
|
|
2454
|
+
Dict mapping file paths to modified source code
|
|
2455
|
+
"""
|
|
2456
|
+
# Group replacements by file
|
|
2457
|
+
replacements_by_file: Dict[str, List[Replacement]] = {}
|
|
2458
|
+
for repl in proposal.replacements:
|
|
2459
|
+
file_path = repl.file_path or proposal.file_path
|
|
2460
|
+
replacements_by_file.setdefault(file_path, []).append(repl)
|
|
2461
|
+
|
|
2462
|
+
# Ensure canonical file is always processed so extracted helper is emitted
|
|
2463
|
+
replacements_by_file.setdefault(proposal.file_path, [])
|
|
2464
|
+
|
|
2465
|
+
# Determine helper names ONCE to avoid mismatches across files
|
|
2466
|
+
# Capture the original helper name before any renaming, and compute the final name
|
|
2467
|
+
original_helper_name = proposal.extracted_function.name
|
|
2468
|
+
if original_helper_name == "__extracted_func":
|
|
2469
|
+
proposal.extracted_function.name = self._allocate_helper_name(proposal.file_path)
|
|
2470
|
+
elif proposal.insert_into_class and not original_helper_name.startswith("_"):
|
|
2471
|
+
# Preserve user-provided helper names but keep them non-public inside classes
|
|
2472
|
+
proposal.extracted_function.name = f"_{original_helper_name}"
|
|
2473
|
+
final_func_name = proposal.extracted_function.name
|
|
2474
|
+
|
|
2475
|
+
# Process each file
|
|
2476
|
+
modified_files: Dict[str, str] = {}
|
|
2477
|
+
|
|
2478
|
+
for file_path, replacements in replacements_by_file.items():
|
|
2479
|
+
# Read original source
|
|
2480
|
+
with open(file_path, "r", encoding="utf-8") as f:
|
|
2481
|
+
lines = f.readlines()
|
|
2482
|
+
|
|
2483
|
+
# Sort replacements by line number (reverse order)
|
|
2484
|
+
replacements = sorted(replacements, key=lambda r: r.line_range[0], reverse=True)
|
|
2485
|
+
|
|
2486
|
+
# Apply each replacement (reverse sorted prevents earlier line shifts)
|
|
2487
|
+
for repl in replacements:
|
|
2488
|
+
start_line, end_line = repl.line_range
|
|
2489
|
+
replacement_node = copy.deepcopy(repl.node)
|
|
2490
|
+
class_name = repl.class_name
|
|
2491
|
+
|
|
2492
|
+
# Validate line numbers
|
|
2493
|
+
if start_line < 1 or start_line > len(lines):
|
|
2494
|
+
print(
|
|
2495
|
+
f"Warning: Invalid line range {start_line}-{end_line} for {file_path} (file has {len(lines)} lines)"
|
|
2496
|
+
)
|
|
2497
|
+
print("Skipping this replacement")
|
|
2498
|
+
continue
|
|
2499
|
+
if end_line > len(lines):
|
|
2500
|
+
print(
|
|
2501
|
+
f"Warning: End line {end_line} exceeds file length {len(lines)} for {file_path}"
|
|
2502
|
+
)
|
|
2503
|
+
print("Adjusting to end of file")
|
|
2504
|
+
end_line = len(lines)
|
|
2505
|
+
|
|
2506
|
+
# Rewrite call sites for method extraction
|
|
2507
|
+
if proposal.insert_into_class and class_name:
|
|
2508
|
+
replacement_node = self._rewrite_call_for_method(
|
|
2509
|
+
replacement_node,
|
|
2510
|
+
original_helper_name,
|
|
2511
|
+
final_func_name,
|
|
2512
|
+
repl.method_kind or proposal.method_kind,
|
|
2513
|
+
repl.implicit_param or proposal.method_param_name,
|
|
2514
|
+
class_name,
|
|
2515
|
+
)
|
|
2516
|
+
elif proposal.insert_into_function:
|
|
2517
|
+
# Rewrite to local function call (no attribute), but ensure call uses final_func_name
|
|
2518
|
+
# Simple textual AST rewrite: replace original helper name with final
|
|
2519
|
+
if original_helper_name != final_func_name:
|
|
2520
|
+
replacement_node = self._retarget_helper_calls(
|
|
2521
|
+
replacement_node,
|
|
2522
|
+
original_helper_name,
|
|
2523
|
+
final_func_name,
|
|
2524
|
+
)
|
|
2525
|
+
else:
|
|
2526
|
+
# Module-level insertion:
|
|
2527
|
+
# If the helper was renamed (e.g., from '__extracted_func' to 'extracted_func' or custom),
|
|
2528
|
+
# rewrite call sites accordingly.
|
|
2529
|
+
if original_helper_name != final_func_name:
|
|
2530
|
+
replacement_node = self._retarget_helper_calls(
|
|
2531
|
+
replacement_node,
|
|
2532
|
+
original_helper_name,
|
|
2533
|
+
final_func_name,
|
|
2534
|
+
)
|
|
2535
|
+
|
|
2536
|
+
replacement_code = ast.unparse(replacement_node)
|
|
2537
|
+
code_lines = replacement_code.split("\n")
|
|
2538
|
+
indent = self._get_indent(lines[start_line - 1])
|
|
2539
|
+
|
|
2540
|
+
# Build indented replacement block: indent ALL non-empty lines consistently
|
|
2541
|
+
replacement_lines: List[str] = []
|
|
2542
|
+
for line in code_lines:
|
|
2543
|
+
if line.strip():
|
|
2544
|
+
replacement_lines.append(indent + line + "\n")
|
|
2545
|
+
else:
|
|
2546
|
+
replacement_lines.append("\n")
|
|
2547
|
+
|
|
2548
|
+
# Splice into source
|
|
2549
|
+
lines[start_line - 1 : end_line] = replacement_lines
|
|
2550
|
+
|
|
2551
|
+
# Insert helper into canonical file or import into others
|
|
2552
|
+
if file_path == proposal.file_path:
|
|
2553
|
+
if proposal.insert_into_function:
|
|
2554
|
+
fn_insert_info = self._find_function_insert_position_before_body_statements(
|
|
2555
|
+
"".join(lines), proposal.insert_into_function
|
|
2556
|
+
)
|
|
2557
|
+
fn_code = ast.unparse(proposal.extracted_function)
|
|
2558
|
+
fn_lines = [l + "\n" for l in fn_code.split("\n")]
|
|
2559
|
+
if fn_insert_info is None:
|
|
2560
|
+
insert_line = self._find_insert_position(lines)
|
|
2561
|
+
lines[insert_line:insert_line] = fn_lines + ["\n", "\n"]
|
|
2562
|
+
else:
|
|
2563
|
+
insert_at_zero_based, indent = fn_insert_info
|
|
2564
|
+
inner_indent = indent + " "
|
|
2565
|
+
indented: List[str] = []
|
|
2566
|
+
for line in fn_lines:
|
|
2567
|
+
if line.strip():
|
|
2568
|
+
indented.append(inner_indent + line)
|
|
2569
|
+
else:
|
|
2570
|
+
indented.append(line)
|
|
2571
|
+
prefix: List[str] = []
|
|
2572
|
+
if insert_at_zero_based > 0 and lines[insert_at_zero_based - 1].strip():
|
|
2573
|
+
prefix.append("\n")
|
|
2574
|
+
suffix: List[str] = []
|
|
2575
|
+
if (
|
|
2576
|
+
insert_at_zero_based < len(lines)
|
|
2577
|
+
and lines[insert_at_zero_based].strip()
|
|
2578
|
+
):
|
|
2579
|
+
suffix.append("\n")
|
|
2580
|
+
lines[insert_at_zero_based:insert_at_zero_based] = (
|
|
2581
|
+
prefix + indented + suffix
|
|
2582
|
+
)
|
|
2583
|
+
elif proposal.insert_into_class:
|
|
2584
|
+
# Prepare method signature & decorator
|
|
2585
|
+
method_kind = proposal.method_kind or "instance"
|
|
2586
|
+
self._prepare_extracted_method_signature(
|
|
2587
|
+
proposal.extracted_function,
|
|
2588
|
+
method_kind,
|
|
2589
|
+
proposal.method_param_name,
|
|
2590
|
+
)
|
|
2591
|
+
func_code = ast.unparse(proposal.extracted_function)
|
|
2592
|
+
method_lines = [line + "\n" for line in func_code.split("\n")]
|
|
2593
|
+
insert_info = self._find_class_insert_position(
|
|
2594
|
+
"".join(lines), proposal.insert_into_class
|
|
2595
|
+
)
|
|
2596
|
+
if insert_info is None:
|
|
2597
|
+
insert_line = self._find_insert_position(lines)
|
|
2598
|
+
lines[insert_line:insert_line] = method_lines + ["\n", "\n"]
|
|
2599
|
+
else:
|
|
2600
|
+
insert_line_zero_based, indent = insert_info
|
|
2601
|
+
method_indent = indent + " "
|
|
2602
|
+
indented_method: List[str] = []
|
|
2603
|
+
for line in method_lines:
|
|
2604
|
+
if line.strip():
|
|
2605
|
+
indented_method.append(method_indent + line)
|
|
2606
|
+
else:
|
|
2607
|
+
indented_method.append(line)
|
|
2608
|
+
insert_at = insert_line_zero_based + 1
|
|
2609
|
+
method_prefix: List[str] = []
|
|
2610
|
+
if insert_at > 0 and lines[insert_at - 1].strip():
|
|
2611
|
+
method_prefix.append("\n")
|
|
2612
|
+
method_suffix: List[str] = []
|
|
2613
|
+
if insert_at < len(lines) and lines[insert_at].strip():
|
|
2614
|
+
method_suffix.append("\n")
|
|
2615
|
+
lines[insert_at:insert_at] = method_prefix + indented_method + method_suffix
|
|
2616
|
+
else:
|
|
2617
|
+
func_code = ast.unparse(proposal.extracted_function)
|
|
2618
|
+
func_lines = [line + "\n" for line in func_code.split("\n")]
|
|
2619
|
+
insert_line = self._find_insert_position(lines)
|
|
2620
|
+
lines_to_insert: List[str] = []
|
|
2621
|
+
if insert_line > 0:
|
|
2622
|
+
blank_lines_before = 0
|
|
2623
|
+
check_line = insert_line - 1
|
|
2624
|
+
while check_line >= 0 and not lines[check_line].strip():
|
|
2625
|
+
blank_lines_before += 1
|
|
2626
|
+
check_line -= 1
|
|
2627
|
+
if blank_lines_before < 2:
|
|
2628
|
+
lines_to_insert.extend(["\n"] * (2 - blank_lines_before))
|
|
2629
|
+
lines_to_insert.extend(func_lines)
|
|
2630
|
+
lines_to_insert.extend(["\n", "\n"])
|
|
2631
|
+
lines[insert_line:insert_line] = lines_to_insert
|
|
2632
|
+
else:
|
|
2633
|
+
# Non-canonical file: insert import (module-level only)
|
|
2634
|
+
if not proposal.insert_into_class:
|
|
2635
|
+
from_path = Path(proposal.file_path)
|
|
2636
|
+
to_path = Path(file_path)
|
|
2637
|
+
from pathlib import Path as _P
|
|
2638
|
+
|
|
2639
|
+
common_dir = _P(
|
|
2640
|
+
__import__("os").path.commonpath([str(from_path), str(to_path)])
|
|
2641
|
+
)
|
|
2642
|
+
layout = ProjectLayout.discover(
|
|
2643
|
+
common_dir,
|
|
2644
|
+
prefer_absolute_imports=self.prefer_absolute_imports,
|
|
2645
|
+
pep420_namespace_packages=self.pep420_namespace_packages,
|
|
2646
|
+
)
|
|
2647
|
+
abs_mod = layout.module_name_for(from_path)
|
|
2648
|
+
if abs_mod and layout.prefer_absolute_imports:
|
|
2649
|
+
module_name = abs_mod
|
|
2650
|
+
else:
|
|
2651
|
+
if from_path.parent == to_path.parent:
|
|
2652
|
+
module_name = from_path.stem
|
|
2653
|
+
else:
|
|
2654
|
+
module_name = abs_mod or from_path.stem
|
|
2655
|
+
func_name = proposal.extracted_function.name
|
|
2656
|
+
import_line = f"from {module_name} import {func_name}\n"
|
|
2657
|
+
if not any(import_line.strip() == ln.strip() for ln in lines):
|
|
2658
|
+
import_pos = self._find_import_position(lines)
|
|
2659
|
+
lines.insert(import_pos, import_line)
|
|
2660
|
+
|
|
2661
|
+
modified_files[file_path] = "".join(lines)
|
|
2662
|
+
|
|
2663
|
+
return modified_files
|
|
2664
|
+
|
|
2665
|
+
def _find_import_position(self, lines: List[str]) -> int:
|
|
2666
|
+
"""Find position to insert a new import statement."""
|
|
2667
|
+
last_import_line, after_docstring = self._scan_module_docstring_and_imports(lines)
|
|
2668
|
+
return last_import_line if last_import_line > 0 else after_docstring
|
|
2669
|
+
|
|
2670
|
+
def _get_indent(self, line: str) -> str:
|
|
2671
|
+
"""Get the indentation of a line."""
|
|
2672
|
+
return line[: len(line) - len(line.lstrip())]
|
|
2673
|
+
|
|
2674
|
+
def _find_insert_position(self, lines: List[str]) -> int:
|
|
2675
|
+
"""
|
|
2676
|
+
Find position to insert extracted function at END of file.
|
|
2677
|
+
|
|
2678
|
+
This prevents line number shifts when multiple refactorings are applied,
|
|
2679
|
+
making sequential refactorings more robust.
|
|
2680
|
+
"""
|
|
2681
|
+
# Insert at the end of the file
|
|
2682
|
+
# Find the last non-blank line
|
|
2683
|
+
for i in range(len(lines) - 1, -1, -1):
|
|
2684
|
+
if lines[i].strip():
|
|
2685
|
+
# Insert after this line
|
|
2686
|
+
return i + 1
|
|
2687
|
+
|
|
2688
|
+
# Empty file - insert at beginning
|
|
2689
|
+
return 0
|
|
2690
|
+
|
|
2691
|
+
def _find_class_insert_position(
|
|
2692
|
+
self, source: str, class_name: str
|
|
2693
|
+
) -> Optional[Tuple[int, str]]:
|
|
2694
|
+
"""
|
|
2695
|
+
Find insertion position (0-based line index) at end of class body and class indentation.
|
|
2696
|
+
|
|
2697
|
+
Returns (insert_line_index, class_indent_str) or None.
|
|
2698
|
+
"""
|
|
2699
|
+
try:
|
|
2700
|
+
tree = ast.parse(source)
|
|
2701
|
+
except SyntaxError:
|
|
2702
|
+
return None
|
|
2703
|
+
|
|
2704
|
+
locator = ClassLocator(source, class_name)
|
|
2705
|
+
locator.visit(tree)
|
|
2706
|
+
return locator.result
|
|
2707
|
+
|
|
2708
|
+
def _find_function_insert_position_before_body_statements(
|
|
2709
|
+
self, source: str, function_name: str
|
|
2710
|
+
) -> Optional[Tuple[int, str]]:
|
|
2711
|
+
"""
|
|
2712
|
+
Find an insertion position (0-based line index) inside the given function BEFORE
|
|
2713
|
+
executable body statements (i.e., after any docstring and after any leading
|
|
2714
|
+
nested defs), along with the function's indentation.
|
|
2715
|
+
|
|
2716
|
+
This ensures the inserted helper is bound before returns/calls are executed.
|
|
2717
|
+
Returns (insert_line_index, function_indent_str) or None if function not found.
|
|
2718
|
+
"""
|
|
2719
|
+
try:
|
|
2720
|
+
tree = ast.parse(source)
|
|
2721
|
+
except SyntaxError:
|
|
2722
|
+
return None
|
|
2723
|
+
|
|
2724
|
+
locator = FuncLocator(source, function_name)
|
|
2725
|
+
locator.visit(tree)
|
|
2726
|
+
return locator.result
|
|
2727
|
+
|
|
2728
|
+
def _get_block_indices(
|
|
2729
|
+
self, function: FunctionNode, block_nodes: List[ast.AST]
|
|
2730
|
+
) -> Optional[Tuple[int, int]]:
|
|
2731
|
+
"""
|
|
2732
|
+
Find the indices of a block within a function body.
|
|
2733
|
+
|
|
2734
|
+
Args:
|
|
2735
|
+
function: Function definition
|
|
2736
|
+
block_nodes: Block to find
|
|
2737
|
+
|
|
2738
|
+
Returns:
|
|
2739
|
+
(start_index, end_index) or None if not found
|
|
2740
|
+
"""
|
|
2741
|
+
if not block_nodes:
|
|
2742
|
+
return None
|
|
2743
|
+
|
|
2744
|
+
# Get function body (skip docstring)
|
|
2745
|
+
body = self._body_without_docstring(function.body)
|
|
2746
|
+
|
|
2747
|
+
# Match by line numbers
|
|
2748
|
+
first_node = cast(Union[ast.stmt, ast.expr], block_nodes[0])
|
|
2749
|
+
last_node = cast(Union[ast.stmt, ast.expr], block_nodes[-1])
|
|
2750
|
+
block_start_line = first_node.lineno
|
|
2751
|
+
block_end_line = (
|
|
2752
|
+
last_node.end_lineno
|
|
2753
|
+
if hasattr(last_node, "end_lineno") and last_node.end_lineno is not None
|
|
2754
|
+
else last_node.lineno
|
|
2755
|
+
)
|
|
2756
|
+
|
|
2757
|
+
# Find matching range in body
|
|
2758
|
+
for i, stmt in enumerate(body):
|
|
2759
|
+
stmt_start = stmt.lineno
|
|
2760
|
+
stmt_end = stmt.end_lineno if hasattr(stmt, "end_lineno") else stmt.lineno
|
|
2761
|
+
|
|
2762
|
+
if stmt_start == block_start_line:
|
|
2763
|
+
# Found start, now find end
|
|
2764
|
+
for j in range(i, len(body)):
|
|
2765
|
+
stmt_end = (
|
|
2766
|
+
body[j].end_lineno if hasattr(body[j], "end_lineno") else body[j].lineno
|
|
2767
|
+
)
|
|
2768
|
+
if stmt_end == block_end_line:
|
|
2769
|
+
return (i, j)
|
|
2770
|
+
|
|
2771
|
+
return None
|
|
2772
|
+
|
|
2773
|
+
def _are_structurally_similar(
|
|
2774
|
+
self,
|
|
2775
|
+
block1: List[ast.AST],
|
|
2776
|
+
block2: List[ast.AST],
|
|
2777
|
+
threshold: float = DEFAULT_SIMILARITY_THRESHOLD,
|
|
2778
|
+
) -> bool:
|
|
2779
|
+
"""
|
|
2780
|
+
Check if two blocks are structurally similar enough to attempt unification.
|
|
2781
|
+
|
|
2782
|
+
This does a rough structural comparison to filter out obviously different blocks.
|
|
2783
|
+
|
|
2784
|
+
Args:
|
|
2785
|
+
block1: First block
|
|
2786
|
+
block2: Second block
|
|
2787
|
+
threshold: Similarity threshold (0.0 to 1.0, default: 0.6)
|
|
2788
|
+
|
|
2789
|
+
Returns:
|
|
2790
|
+
True if blocks are similar enough
|
|
2791
|
+
"""
|
|
2792
|
+
if len(block1) != len(block2):
|
|
2793
|
+
return False
|
|
2794
|
+
|
|
2795
|
+
total_nodes = 0
|
|
2796
|
+
matching_nodes = 0
|
|
2797
|
+
|
|
2798
|
+
for stmt1, stmt2 in zip(block1, block2):
|
|
2799
|
+
# Compare AST structure
|
|
2800
|
+
nodes1 = list(ast.walk(stmt1))
|
|
2801
|
+
nodes2 = list(ast.walk(stmt2))
|
|
2802
|
+
|
|
2803
|
+
# Must have similar number of nodes
|
|
2804
|
+
if abs(len(nodes1) - len(nodes2)) / max(len(nodes1), len(nodes2)) > 0.3:
|
|
2805
|
+
return False
|
|
2806
|
+
|
|
2807
|
+
# Count matching node types
|
|
2808
|
+
types1 = [type(n).__name__ for n in nodes1]
|
|
2809
|
+
types2 = [type(n).__name__ for n in nodes2]
|
|
2810
|
+
|
|
2811
|
+
# Count common types
|
|
2812
|
+
from collections import Counter
|
|
2813
|
+
|
|
2814
|
+
counter1 = Counter(types1)
|
|
2815
|
+
counter2 = Counter(types2)
|
|
2816
|
+
|
|
2817
|
+
common = sum((counter1 & counter2).values())
|
|
2818
|
+
total = max(len(types1), len(types2))
|
|
2819
|
+
|
|
2820
|
+
total_nodes += total
|
|
2821
|
+
matching_nodes += common
|
|
2822
|
+
|
|
2823
|
+
if total_nodes == 0:
|
|
2824
|
+
return False
|
|
2825
|
+
|
|
2826
|
+
similarity = matching_nodes / total_nodes
|
|
2827
|
+
return similarity >= threshold
|
|
2828
|
+
|
|
2829
|
+
def refactor_to_fixed_point(
|
|
2830
|
+
self, file_path: str, max_iterations: int = 10
|
|
2831
|
+
) -> Tuple[str, int, List[str]]:
|
|
2832
|
+
"""
|
|
2833
|
+
Apply refactorings iteratively until a fixed point is reached.
|
|
2834
|
+
|
|
2835
|
+
This method applies refactorings one at a time, re-analyzing after each
|
|
2836
|
+
application. This prevents the sequential corruption bug where applying
|
|
2837
|
+
multiple refactorings at once causes line number misalignment.
|
|
2838
|
+
|
|
2839
|
+
max_iterations semantics:
|
|
2840
|
+
- If max_iterations > 0, stop after at most that many applied refactorings.
|
|
2841
|
+
- If max_iterations <= 0, run until a natural fixed point (no proposals found).
|
|
2842
|
+
|
|
2843
|
+
Args:
|
|
2844
|
+
file_path: Path to file to refactor
|
|
2845
|
+
max_iterations: Maximum iterations to prevent infinite loops
|
|
2846
|
+
|
|
2847
|
+
Returns:
|
|
2848
|
+
Tuple of (final_code, num_refactorings_applied, descriptions)
|
|
2849
|
+
"""
|
|
2850
|
+
current_code = Path(file_path).read_text()
|
|
2851
|
+
num_applied = 0
|
|
2852
|
+
descriptions = []
|
|
2853
|
+
|
|
2854
|
+
iteration = 0
|
|
2855
|
+
while True:
|
|
2856
|
+
# Write current code to file for analysis
|
|
2857
|
+
with open(file_path, "w") as f:
|
|
2858
|
+
f.write(current_code)
|
|
2859
|
+
|
|
2860
|
+
# Analyze for refactoring opportunities
|
|
2861
|
+
# Important: invalidate cached analysis for this path so we see latest edits
|
|
2862
|
+
proposals = self.analyze_files([file_path], invalidate_paths=[file_path])
|
|
2863
|
+
|
|
2864
|
+
if not proposals:
|
|
2865
|
+
# Fixed point reached - no more refactorings found
|
|
2866
|
+
break
|
|
2867
|
+
|
|
2868
|
+
# Apply only the first proposal
|
|
2869
|
+
proposal = proposals[0]
|
|
2870
|
+
new_code = self.apply_refactoring(file_path, proposal)
|
|
2871
|
+
# Idempotence guard: if no change, stop to avoid churn
|
|
2872
|
+
if new_code == current_code:
|
|
2873
|
+
break
|
|
2874
|
+
current_code = new_code
|
|
2875
|
+
num_applied += 1
|
|
2876
|
+
descriptions.append(proposal.description)
|
|
2877
|
+
|
|
2878
|
+
iteration += 1
|
|
2879
|
+
if max_iterations > 0 and iteration >= max_iterations:
|
|
2880
|
+
break
|
|
2881
|
+
|
|
2882
|
+
return current_code, num_applied, descriptions
|
|
2883
|
+
|
|
2884
|
+
def refactor_directory_to_fixed_point(
|
|
2885
|
+
self,
|
|
2886
|
+
input_dir: str,
|
|
2887
|
+
output_dir: str,
|
|
2888
|
+
max_iterations: int = 10,
|
|
2889
|
+
progress: str = "tqdm",
|
|
2890
|
+
) -> Tuple[Dict[str, Tuple[int, List[str]]], str]:
|
|
2891
|
+
"""
|
|
2892
|
+
Apply refactorings across a directory (recursively) until a fixed point.
|
|
2893
|
+
|
|
2894
|
+
Unlike the per-file variant, this performs whole-project analysis on every
|
|
2895
|
+
iteration so it can apply BOTH same-file and cross-file proposals. One proposal
|
|
2896
|
+
is applied per iteration, then the directory is re-analyzed, up to max_iterations.
|
|
2897
|
+
|
|
2898
|
+
Args:
|
|
2899
|
+
input_dir: Input directory path
|
|
2900
|
+
output_dir: Output directory path (results are written here)
|
|
2901
|
+
max_iterations: Maximum iterations to prevent infinite loops. If <= 0,
|
|
2902
|
+
run until a natural fixed point (no proposals remain).
|
|
2903
|
+
progress: Progress display mode: 'tqdm' for percentage bar if available,
|
|
2904
|
+
'auto' fallback to simple inline bar, 'none' disables progress output.
|
|
2905
|
+
|
|
2906
|
+
Returns:
|
|
2907
|
+
(results_dict, termination_reason)
|
|
2908
|
+
termination_reason ∈ {"fixed_point", "iteration_cap"}
|
|
2909
|
+
"""
|
|
2910
|
+
from pathlib import Path
|
|
2911
|
+
import shutil
|
|
2912
|
+
import textwrap
|
|
2913
|
+
|
|
2914
|
+
progress_mode, tqdm_wrapper, use_tqdm = self._resolve_progress_backend(progress)
|
|
2915
|
+
|
|
2916
|
+
input_path = Path(input_dir)
|
|
2917
|
+
output_path = Path(output_dir)
|
|
2918
|
+
|
|
2919
|
+
# Create output directory
|
|
2920
|
+
output_path.mkdir(parents=True, exist_ok=True)
|
|
2921
|
+
|
|
2922
|
+
# Copy all files to output directory first (skip if same path)
|
|
2923
|
+
if input_path != output_path:
|
|
2924
|
+
for item in input_path.rglob("*"):
|
|
2925
|
+
if item.is_file():
|
|
2926
|
+
rel_path = item.relative_to(input_path)
|
|
2927
|
+
output_file = output_path / rel_path
|
|
2928
|
+
output_file.parent.mkdir(parents=True, exist_ok=True)
|
|
2929
|
+
shutil.copy2(item, output_file)
|
|
2930
|
+
|
|
2931
|
+
# Aggregate results per file
|
|
2932
|
+
results: Dict[str, Tuple[int, List[str]]] = {}
|
|
2933
|
+
|
|
2934
|
+
def _bump_result(path: str, desc: str) -> None:
|
|
2935
|
+
count, descs = results.get(path, (0, []))
|
|
2936
|
+
results[path] = (count + 1, descs + [desc])
|
|
2937
|
+
|
|
2938
|
+
# Proposal processing state
|
|
2939
|
+
proposal_queue: List[RefactoringProposal] = []
|
|
2940
|
+
iterations = 0
|
|
2941
|
+
total_applied = 0
|
|
2942
|
+
termination_reason = "fixed_point"
|
|
2943
|
+
|
|
2944
|
+
# Timing / ETA state (for heuristic ETA when total unknown)
|
|
2945
|
+
import time as _time
|
|
2946
|
+
|
|
2947
|
+
per_proposal_durations: List[float] = []
|
|
2948
|
+
|
|
2949
|
+
# Progress helpers -------------------------------------------------
|
|
2950
|
+
progress_bar = None
|
|
2951
|
+
|
|
2952
|
+
# Fallback inline bar (only when not using tqdm and not in detail/none)
|
|
2953
|
+
def _fallback_bar(applied: int, queued: int, phase: str, desc: str) -> None:
|
|
2954
|
+
# Suppress inline fallback bar when tqdm is selected or active, or in 'none'/'detail' modes
|
|
2955
|
+
if progress_mode in ("none", "detail", "tqdm") or use_tqdm:
|
|
2956
|
+
return
|
|
2957
|
+
try:
|
|
2958
|
+
denom = max(applied + queued, 1)
|
|
2959
|
+
pct = int((applied / denom) * 100)
|
|
2960
|
+
bar = self._render_inline_bar(pct, bar_len=32)
|
|
2961
|
+
short = desc if len(desc) <= 48 else desc[:45] + "..."
|
|
2962
|
+
print(
|
|
2963
|
+
f"\r[towel] {phase:<10} [{bar}] {pct:3d}% | applied={applied} queued={queued} | {short}",
|
|
2964
|
+
end="",
|
|
2965
|
+
flush=True,
|
|
2966
|
+
)
|
|
2967
|
+
except Exception:
|
|
2968
|
+
pass
|
|
2969
|
+
|
|
2970
|
+
def _update_progress_postfix(applied: int, queued: int) -> None:
|
|
2971
|
+
"""Keep tqdm postfix updates consistent (see docs/DRY_RUN_2025-11-28.md)."""
|
|
2972
|
+
if not (use_tqdm and progress_bar is not None):
|
|
2973
|
+
return
|
|
2974
|
+
try:
|
|
2975
|
+
progress_bar.set_postfix({"A": applied, "Q": queued}, refresh=True)
|
|
2976
|
+
except Exception:
|
|
2977
|
+
pass
|
|
2978
|
+
|
|
2979
|
+
def _apply_proposal_and_refresh_queue(
|
|
2980
|
+
proposal: RefactoringProposal, queue: List[RefactoringProposal]
|
|
2981
|
+
) -> List[RefactoringProposal]:
|
|
2982
|
+
"""Apply proposal, update caches, and refresh queue (DRY helper)."""
|
|
2983
|
+
result = self.apply_refactoring_multi_file(proposal)
|
|
2984
|
+
if isinstance(result, tuple):
|
|
2985
|
+
modified_files, changed_paths = result
|
|
2986
|
+
else:
|
|
2987
|
+
modified_files = result
|
|
2988
|
+
changed_paths = list(modified_files.keys())
|
|
2989
|
+
|
|
2990
|
+
for fpath, content in modified_files.items():
|
|
2991
|
+
with open(fpath, "w", encoding="utf-8") as fh:
|
|
2992
|
+
fh.write(content)
|
|
2993
|
+
_bump_result(fpath, proposal.description)
|
|
2994
|
+
|
|
2995
|
+
if hasattr(self, "invalidate_paths") and changed_paths:
|
|
2996
|
+
try:
|
|
2997
|
+
self.invalidate_paths(changed_paths)
|
|
2998
|
+
except Exception:
|
|
2999
|
+
pass
|
|
3000
|
+
|
|
3001
|
+
if changed_paths:
|
|
3002
|
+
changed_set = set(map(str, changed_paths))
|
|
3003
|
+
queue = [
|
|
3004
|
+
p
|
|
3005
|
+
for p in queue
|
|
3006
|
+
if not any(
|
|
3007
|
+
(rep.file_path or p.file_path) in changed_set for rep in p.replacements
|
|
3008
|
+
)
|
|
3009
|
+
]
|
|
3010
|
+
|
|
3011
|
+
try:
|
|
3012
|
+
localized = self.analyze_files(
|
|
3013
|
+
list(changed_paths), invalidate_paths=list(changed_paths)
|
|
3014
|
+
)
|
|
3015
|
+
if localized:
|
|
3016
|
+
localized = filter_overlapping_proposals(localized)
|
|
3017
|
+
localized = [
|
|
3018
|
+
p
|
|
3019
|
+
for p in localized
|
|
3020
|
+
if any(
|
|
3021
|
+
(rep.file_path or p.file_path) in changed_set
|
|
3022
|
+
for rep in p.replacements
|
|
3023
|
+
)
|
|
3024
|
+
]
|
|
3025
|
+
if localized:
|
|
3026
|
+
queue = localized + queue
|
|
3027
|
+
_detail(f"Localized +{len(localized)} follow-up(s)")
|
|
3028
|
+
_fallback_bar(
|
|
3029
|
+
total_applied,
|
|
3030
|
+
len(queue),
|
|
3031
|
+
"localized",
|
|
3032
|
+
f"+{len(localized)} follow-ups",
|
|
3033
|
+
)
|
|
3034
|
+
except Exception:
|
|
3035
|
+
pass
|
|
3036
|
+
|
|
3037
|
+
return queue
|
|
3038
|
+
|
|
3039
|
+
# Note: We defer tqdm progress bar creation until we have proposals to apply.
|
|
3040
|
+
# This avoids an early line like "analyzing: 0it" with unknown totals.
|
|
3041
|
+
|
|
3042
|
+
def _detail(msg: str) -> None:
|
|
3043
|
+
if progress_mode == "detail":
|
|
3044
|
+
print(f"[towel] {msg}")
|
|
3045
|
+
|
|
3046
|
+
# Main loop -------------------------------------------------------
|
|
3047
|
+
while True:
|
|
3048
|
+
if not proposal_queue:
|
|
3049
|
+
# Global analysis pass
|
|
3050
|
+
if progress_mode != "none":
|
|
3051
|
+
try:
|
|
3052
|
+
file_count = sum(1 for _ in output_path.rglob("*.py"))
|
|
3053
|
+
_detail(f"Analyzing {file_count} file(s)...")
|
|
3054
|
+
except Exception:
|
|
3055
|
+
pass
|
|
3056
|
+
# Show pairing progress during global analysis if user requested progress bars.
|
|
3057
|
+
analysis_progress_flag = (
|
|
3058
|
+
progress_mode if progress_mode in ("tqdm", "auto") else "none"
|
|
3059
|
+
)
|
|
3060
|
+
proposals = self.analyze_directory(
|
|
3061
|
+
str(output_path), recursive=True, verbose=False, progress=analysis_progress_flag
|
|
3062
|
+
)
|
|
3063
|
+
if not proposals:
|
|
3064
|
+
# Fixed point reached
|
|
3065
|
+
if use_tqdm and progress_bar is not None:
|
|
3066
|
+
# Ensure a clean newline so the last line doesn't meld with following prints
|
|
3067
|
+
try:
|
|
3068
|
+
progress_bar.refresh()
|
|
3069
|
+
progress_bar.close()
|
|
3070
|
+
except Exception:
|
|
3071
|
+
pass
|
|
3072
|
+
else:
|
|
3073
|
+
if progress_mode not in ("none", "detail") and not use_tqdm:
|
|
3074
|
+
print() # finish inline bar line
|
|
3075
|
+
break
|
|
3076
|
+
proposal_queue = filter_overlapping_proposals(proposals)
|
|
3077
|
+
_detail(f"Discovered {len(proposal_queue)} proposal(s)")
|
|
3078
|
+
if progress_mode == "detail":
|
|
3079
|
+
for i, p in enumerate(proposal_queue[:25], 1): # cap verbose listing
|
|
3080
|
+
short = textwrap.shorten(p.description, width=100, placeholder="...")
|
|
3081
|
+
print(f" {i:2d}. {short}")
|
|
3082
|
+
if len(proposal_queue) > 25:
|
|
3083
|
+
print(f" ... {len(proposal_queue)-25} more")
|
|
3084
|
+
_fallback_bar(total_applied, len(proposal_queue), "discovered", "proposals queued")
|
|
3085
|
+
if use_tqdm and progress_bar is None:
|
|
3086
|
+
# Lazily create tqdm now that we have proposals to apply
|
|
3087
|
+
assert tqdm_wrapper is not None
|
|
3088
|
+
try:
|
|
3089
|
+
total_known = max_iterations > 0
|
|
3090
|
+
# Use leave=False so subsequent prints don't duplicate the bar line.
|
|
3091
|
+
progress_bar = tqdm_wrapper(
|
|
3092
|
+
total=max_iterations if total_known else None,
|
|
3093
|
+
desc="apply",
|
|
3094
|
+
unit="it",
|
|
3095
|
+
dynamic_ncols=True,
|
|
3096
|
+
leave=False,
|
|
3097
|
+
)
|
|
3098
|
+
queued_ct = len(proposal_queue)
|
|
3099
|
+
_update_progress_postfix(0, queued_ct)
|
|
3100
|
+
except Exception:
|
|
3101
|
+
progress_bar = None
|
|
3102
|
+
|
|
3103
|
+
if not proposal_queue:
|
|
3104
|
+
break
|
|
3105
|
+
|
|
3106
|
+
proposal = self._pop_next_proposal(proposal_queue)
|
|
3107
|
+
if proposal is None:
|
|
3108
|
+
break
|
|
3109
|
+
iter_start = _time.time()
|
|
3110
|
+
last_desc = proposal.description
|
|
3111
|
+
# Suppress separate applying log line when tqdm active to avoid duplicate lines
|
|
3112
|
+
if not (use_tqdm and progress_bar is not None):
|
|
3113
|
+
_fallback_bar(
|
|
3114
|
+
total_applied, len(proposal_queue), "apply", f"#{iterations+1}: {last_desc}"
|
|
3115
|
+
)
|
|
3116
|
+
|
|
3117
|
+
# Apply proposal
|
|
3118
|
+
proposal_queue = _apply_proposal_and_refresh_queue(proposal, proposal_queue)
|
|
3119
|
+
|
|
3120
|
+
# Record duration for this iteration (include localized follow-up analysis time)
|
|
3121
|
+
per_proposal_durations.append(_time.time() - iter_start)
|
|
3122
|
+
iterations += 1
|
|
3123
|
+
total_applied += 1
|
|
3124
|
+
if use_tqdm and progress_bar is not None:
|
|
3125
|
+
try:
|
|
3126
|
+
progress_bar.update(1)
|
|
3127
|
+
queued_ct = len(proposal_queue)
|
|
3128
|
+
except Exception:
|
|
3129
|
+
pass
|
|
3130
|
+
else:
|
|
3131
|
+
_update_progress_postfix(total_applied, queued_ct)
|
|
3132
|
+
else:
|
|
3133
|
+
_fallback_bar(
|
|
3134
|
+
total_applied, len(proposal_queue), "applied", f"#{iterations}: {last_desc}"
|
|
3135
|
+
)
|
|
3136
|
+
|
|
3137
|
+
if max_iterations > 0 and iterations >= max_iterations:
|
|
3138
|
+
termination_reason = "iteration_cap"
|
|
3139
|
+
if use_tqdm and progress_bar is not None:
|
|
3140
|
+
progress_bar.close()
|
|
3141
|
+
else:
|
|
3142
|
+
if progress_mode not in ("none", "detail") and not use_tqdm:
|
|
3143
|
+
print()
|
|
3144
|
+
break
|
|
3145
|
+
|
|
3146
|
+
return results, termination_reason
|
|
3147
|
+
|
|
3148
|
+
# Optional analysis cache invalidation hook used by directory fixed-point runner
|
|
3149
|
+
def invalidate_paths(self, paths: List[str]) -> None: # pragma: no cover - simple cache hook
|
|
3150
|
+
try:
|
|
3151
|
+
from . import pipeline as _pipeline # local import to avoid cycles at module load time
|
|
3152
|
+
|
|
3153
|
+
for p in paths:
|
|
3154
|
+
if hasattr(_pipeline, "_analysis_cache"):
|
|
3155
|
+
cache = cast(Dict[str, Any], getattr(_pipeline, "_analysis_cache"))
|
|
3156
|
+
cache.pop(p, None)
|
|
3157
|
+
except Exception:
|
|
3158
|
+
# Non-fatal; best-effort invalidation only
|
|
3159
|
+
pass
|
|
3160
|
+
|
|
3161
|
+
|
|
3162
|
+
def _refactor_engine_helper_2(self, all_functions, block_pairs, class_infos, progress, verbose):
|
|
3163
|
+
return self._evaluate_pairs_serial(
|
|
3164
|
+
block_pairs, all_functions, class_infos, verbose=verbose, progress=progress
|
|
3165
|
+
)
|
|
3166
|
+
|
|
3167
|
+
|
|
3168
|
+
# Utility functions for overlap filtering
|
|
3169
|
+
|
|
3170
|
+
|
|
3171
|
+
def get_affected_lines(proposal: RefactoringProposal) -> Set[Tuple[str, int]]:
|
|
3172
|
+
"""
|
|
3173
|
+
Get all (file_path, line_number) tuples affected by a proposal.
|
|
3174
|
+
|
|
3175
|
+
This is used for overlap detection - two proposals overlap if they
|
|
3176
|
+
affect any of the same lines in the same file.
|
|
3177
|
+
|
|
3178
|
+
Args:
|
|
3179
|
+
proposal: A refactoring proposal
|
|
3180
|
+
|
|
3181
|
+
Returns:
|
|
3182
|
+
Set of (file_path, line_number) tuples that would be modified
|
|
3183
|
+
"""
|
|
3184
|
+
affected: Set[Tuple[str, int]] = set()
|
|
3185
|
+
for repl in proposal.replacements:
|
|
3186
|
+
file_path = repl.file_path or proposal.file_path
|
|
3187
|
+
start_line, end_line = repl.line_range
|
|
3188
|
+
for line_num in range(start_line, end_line + 1):
|
|
3189
|
+
affected.add((file_path, line_num))
|
|
3190
|
+
|
|
3191
|
+
return affected
|
|
3192
|
+
|
|
3193
|
+
|
|
3194
|
+
def filter_overlapping_proposals(proposals: List[RefactoringProposal]) -> List[RefactoringProposal]:
|
|
3195
|
+
"""
|
|
3196
|
+
Filter proposals to remove overlaps, keeping the best ones.
|
|
3197
|
+
|
|
3198
|
+
When multiple proposals affect overlapping lines, this function selects
|
|
3199
|
+
a subset of non-overlapping proposals. Larger proposals (affecting more
|
|
3200
|
+
lines) are preferred over smaller ones.
|
|
3201
|
+
|
|
3202
|
+
Strategy:
|
|
3203
|
+
1. Sort proposals by size (total lines affected), largest first
|
|
3204
|
+
2. Greedily select proposals that don't overlap with already-selected ones
|
|
3205
|
+
|
|
3206
|
+
Args:
|
|
3207
|
+
proposals: List of refactoring proposals
|
|
3208
|
+
|
|
3209
|
+
Returns:
|
|
3210
|
+
List of non-overlapping proposals, sorted by size (largest first)
|
|
3211
|
+
|
|
3212
|
+
Example:
|
|
3213
|
+
If proposals affect lines [1-7], [1-6], and [2-7], only [1-7]
|
|
3214
|
+
would be selected as it's the largest and the others overlap with it.
|
|
3215
|
+
"""
|
|
3216
|
+
if not proposals:
|
|
3217
|
+
return []
|
|
3218
|
+
|
|
3219
|
+
def proposal_size(p: RefactoringProposal) -> int:
|
|
3220
|
+
"""Calculate total lines affected by a proposal."""
|
|
3221
|
+
total_lines = 0
|
|
3222
|
+
for repl in p.replacements:
|
|
3223
|
+
start_line, end_line = repl.line_range
|
|
3224
|
+
total_lines += end_line - start_line + 1
|
|
3225
|
+
return total_lines
|
|
3226
|
+
|
|
3227
|
+
# Optional debug diagnostics: env flag
|
|
3228
|
+
import os as _os
|
|
3229
|
+
|
|
3230
|
+
_debug_overlap = bool(_os.getenv("DEBUG_OVERLAP_FILTER"))
|
|
3231
|
+
|
|
3232
|
+
# Helpers for deterministic ordering and interval extraction
|
|
3233
|
+
def first_span(p: RefactoringProposal) -> Tuple[str, int]:
|
|
3234
|
+
if not p.replacements:
|
|
3235
|
+
return (p.file_path, 0)
|
|
3236
|
+
starts = [r.line_range[0] for r in p.replacements]
|
|
3237
|
+
return (p.file_path, min(starts))
|
|
3238
|
+
|
|
3239
|
+
# Map proposals to affected lines and per-file convex-hull intervals
|
|
3240
|
+
prop_affected: Dict[int, Set[Tuple[str, int]]] = {}
|
|
3241
|
+
prop_files: Dict[int, Set[str]] = {}
|
|
3242
|
+
per_file_interval: Dict[int, Dict[str, Tuple[int, int]]] = {}
|
|
3243
|
+
|
|
3244
|
+
for idx, p in enumerate(proposals):
|
|
3245
|
+
affected = get_affected_lines(p)
|
|
3246
|
+
prop_affected[idx] = affected
|
|
3247
|
+
files: Set[str] = set(fp for (fp, _ln) in affected)
|
|
3248
|
+
prop_files[idx] = files
|
|
3249
|
+
per_file_interval[idx] = {}
|
|
3250
|
+
# Build convex hull interval per file for MWIS approximation
|
|
3251
|
+
by_file: Dict[str, List[int]] = {}
|
|
3252
|
+
for fp, ln in affected:
|
|
3253
|
+
by_file.setdefault(fp, []).append(ln)
|
|
3254
|
+
for fp, lines in by_file.items():
|
|
3255
|
+
per_file_interval[idx][fp] = (min(lines), max(lines))
|
|
3256
|
+
|
|
3257
|
+
# Stage 1: Optimal non-overlapping selection within single-file proposals using MWIS
|
|
3258
|
+
selected_indices: Set[int] = set()
|
|
3259
|
+
used_lines: Set[Tuple[str, int]] = set()
|
|
3260
|
+
|
|
3261
|
+
# Group single-file proposals by that file
|
|
3262
|
+
by_primary_file: Dict[str, List[int]] = {}
|
|
3263
|
+
for idx, files in prop_files.items():
|
|
3264
|
+
if len(files) == 1:
|
|
3265
|
+
fp = next(iter(files))
|
|
3266
|
+
by_primary_file.setdefault(fp, []).append(idx)
|
|
3267
|
+
|
|
3268
|
+
def run_weighted_interval_scheduling(file_path: str, indices: List[int]) -> List[int]:
|
|
3269
|
+
# Build items: (start, end, weight, idx)
|
|
3270
|
+
items: List[Tuple[int, int, int, int, Tuple[str, int]]] = []
|
|
3271
|
+
for idx in indices:
|
|
3272
|
+
start, end = per_file_interval[idx][file_path]
|
|
3273
|
+
weight = proposal_size(proposals[idx])
|
|
3274
|
+
items.append((start, end, weight, idx, first_span(proposals[idx])))
|
|
3275
|
+
|
|
3276
|
+
# Sort by end then tie-breaker to stabilize
|
|
3277
|
+
items.sort(key=lambda t: (t[1], t[0], -t[2], t[4]))
|
|
3278
|
+
|
|
3279
|
+
n = len(items)
|
|
3280
|
+
if n == 0:
|
|
3281
|
+
return []
|
|
3282
|
+
|
|
3283
|
+
# Precompute p[j]: rightmost non-overlapping interval index before j
|
|
3284
|
+
ends = [it[1] for it in items]
|
|
3285
|
+
starts = [it[0] for it in items]
|
|
3286
|
+
p = [-1] * n
|
|
3287
|
+
j = 0
|
|
3288
|
+
for j in range(n):
|
|
3289
|
+
# binary search for last i with ends[i] < starts[j]
|
|
3290
|
+
lo, hi = 0, j - 1
|
|
3291
|
+
last = -1
|
|
3292
|
+
while lo <= hi:
|
|
3293
|
+
mid = (lo + hi) // 2
|
|
3294
|
+
if ends[mid] < starts[j]:
|
|
3295
|
+
last = mid
|
|
3296
|
+
lo = mid + 1
|
|
3297
|
+
else:
|
|
3298
|
+
hi = mid - 1
|
|
3299
|
+
p[j] = last
|
|
3300
|
+
|
|
3301
|
+
# DP arrays
|
|
3302
|
+
dp = [0] * n
|
|
3303
|
+
take = [False] * n
|
|
3304
|
+
for j in range(n):
|
|
3305
|
+
wj = items[j][2]
|
|
3306
|
+
without = dp[j - 1] if j > 0 else 0
|
|
3307
|
+
withj = wj + (dp[p[j]] if p[j] != -1 else 0)
|
|
3308
|
+
if withj > without:
|
|
3309
|
+
dp[j] = withj
|
|
3310
|
+
take[j] = True
|
|
3311
|
+
elif withj == without:
|
|
3312
|
+
# Tie-breaker: prefer earlier ending interval set implicitly
|
|
3313
|
+
dp[j] = without
|
|
3314
|
+
take[j] = False
|
|
3315
|
+
else:
|
|
3316
|
+
dp[j] = without
|
|
3317
|
+
take[j] = False
|
|
3318
|
+
|
|
3319
|
+
# Reconstruct
|
|
3320
|
+
sel: List[int] = []
|
|
3321
|
+
j = n - 1
|
|
3322
|
+
while j >= 0:
|
|
3323
|
+
if take[j]:
|
|
3324
|
+
sel.append(items[j][3])
|
|
3325
|
+
j = p[j]
|
|
3326
|
+
else:
|
|
3327
|
+
j -= 1
|
|
3328
|
+
sel.reverse()
|
|
3329
|
+
if _debug_overlap:
|
|
3330
|
+
try:
|
|
3331
|
+
print(
|
|
3332
|
+
f"OVERLAP_OPTIMAL file={file_path} selected={len(sel)} total_weight={dp[n-1]} candidates={n}"
|
|
3333
|
+
)
|
|
3334
|
+
except Exception:
|
|
3335
|
+
pass
|
|
3336
|
+
return sel
|
|
3337
|
+
|
|
3338
|
+
for fp, idxs in by_primary_file.items():
|
|
3339
|
+
chosen = run_weighted_interval_scheduling(fp, idxs)
|
|
3340
|
+
for idx in chosen:
|
|
3341
|
+
if idx not in selected_indices:
|
|
3342
|
+
selected_indices.add(idx)
|
|
3343
|
+
used_lines.update(prop_affected[idx])
|
|
3344
|
+
|
|
3345
|
+
# Stage 2: Greedy add for remaining proposals (multi-file or leftover), respecting used_lines
|
|
3346
|
+
def sort_key(p: RefactoringProposal) -> Tuple[int, Tuple[str, int]]:
|
|
3347
|
+
return (-(proposal_size(p)), first_span(p))
|
|
3348
|
+
|
|
3349
|
+
remaining = [i for i in range(len(proposals)) if i not in selected_indices]
|
|
3350
|
+
remaining_sorted = sorted(remaining, key=lambda i: sort_key(proposals[i]))
|
|
3351
|
+
|
|
3352
|
+
selected: List[RefactoringProposal] = [proposals[i] for i in selected_indices]
|
|
3353
|
+
|
|
3354
|
+
for idx in remaining_sorted:
|
|
3355
|
+
affected = prop_affected[idx]
|
|
3356
|
+
intersection = affected & used_lines
|
|
3357
|
+
if not intersection:
|
|
3358
|
+
selected.append(proposals[idx])
|
|
3359
|
+
used_lines.update(affected)
|
|
3360
|
+
elif _debug_overlap:
|
|
3361
|
+
by_file2: Dict[str, List[int]] = {}
|
|
3362
|
+
for fp, ln in intersection:
|
|
3363
|
+
by_file2.setdefault(fp, []).append(ln)
|
|
3364
|
+
parts = [
|
|
3365
|
+
f"{fp}:{min(lines)}-{max(lines)} ({len(lines)} lines)"
|
|
3366
|
+
for fp, lines in by_file2.items()
|
|
3367
|
+
]
|
|
3368
|
+
try:
|
|
3369
|
+
print(
|
|
3370
|
+
"OVERLAP_DROP: size=",
|
|
3371
|
+
proposal_size(proposals[idx]),
|
|
3372
|
+
" first=",
|
|
3373
|
+
first_span(proposals[idx]),
|
|
3374
|
+
" because intersects ",
|
|
3375
|
+
"; ".join(parts),
|
|
3376
|
+
)
|
|
3377
|
+
except Exception:
|
|
3378
|
+
pass
|
|
3379
|
+
|
|
3380
|
+
# Return selected sorted by size descending for external stability
|
|
3381
|
+
return sorted(selected, key=lambda p: (-(proposal_size(p)), first_span(p)))
|