code-towel 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,3381 @@
1
+ # Copyright 2025 Eric Allen
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """
16
+ Main refactoring engine using unification.
17
+
18
+ This orchestrates the entire refactoring process:
19
+ 1. Parse files into ASTs
20
+ 2. Find pairs of code blocks in top-level functions
21
+ 3. Attempt unification to find parameterizable differences
22
+ 4. Extract functions hygienically if unification succeeds
23
+ 5. Generate replacement calls
24
+ """
25
+
26
+ import ast
27
+ import copy
28
+ import os
29
+ import re
30
+ from collections import deque
31
+ from concurrent.futures import ProcessPoolExecutor, as_completed
32
+ from dataclasses import dataclass
33
+ from typing import Any, List, Tuple, Dict, Set, Optional, FrozenSet, Literal, Union, Sequence, cast
34
+ from weakref import WeakKeyDictionary
35
+ from pathlib import Path
36
+ from .scope_analyzer import ScopeAnalyzer, Scope
37
+ from .unifier import Unifier
38
+ from .extractor import HygienicExtractor, is_value_producing
39
+ from .orphan_detector import has_orphaned_variables
40
+ from .assignment_analyzer import (
41
+ analyze_assignments,
42
+ has_reassignments_without_bindings,
43
+ _collect_block_binding_stats,
44
+ _collect_bindings_and_reassignments,
45
+ )
46
+ from .project_layout import ProjectLayout
47
+ from .block_signature import extract_block_signature, quick_filter
48
+ from .models import (
49
+ CodeBlockPair,
50
+ MethodInfo,
51
+ ClassInfo,
52
+ ClassInsertionPlan,
53
+ Replacement,
54
+ RefactoringProposal,
55
+ )
56
+ from .pipeline import run_pipeline
57
+ from .visitors import (
58
+ MethodCallRewriter,
59
+ LoopReturnFinder,
60
+ NameCollector,
61
+ AugAssignFinder,
62
+ AssignTargetVisitor,
63
+ ClassLocator,
64
+ FuncLocator,
65
+ )
66
+
67
+ # Configuration defaults
68
+ DEFAULT_MAX_PARAMETERS = 5
69
+ DEFAULT_MIN_LINES = 4
70
+ DEFAULT_SIMILARITY_THRESHOLD = 0.6
71
+ DEFAULT_MAX_ITERATIONS = 0 # Unlimited
72
+
73
+ FunctionNode = Union[ast.FunctionDef, ast.AsyncFunctionDef]
74
+
75
+
76
+ @dataclass(frozen=True)
77
+ class BlockBindingSnapshot:
78
+ """Summarized binding data for a block, mirroring DRY helper structure."""
79
+
80
+ bound_in_block: Set[str]
81
+ reassigned_in_block: Set[str]
82
+ bound_before_block: Set[str]
83
+ bound_after_block: Set[str]
84
+ initially_bound: Set[str]
85
+
86
+
87
+ _worker_engine: Optional["UnificationRefactorEngine"] = None
88
+ _worker_functions: Optional[
89
+ List[
90
+ Tuple[
91
+ str,
92
+ FunctionNode,
93
+ str,
94
+ "ScopeAnalyzer",
95
+ "Scope",
96
+ Optional[str],
97
+ Optional[str],
98
+ List[str],
99
+ ]
100
+ ]
101
+ ] = None
102
+ _worker_class_infos: Optional[List[ClassInfo]] = None
103
+
104
+
105
+ def _initialize_pair_worker(
106
+ engine_config: Dict[str, Any],
107
+ functions: List[
108
+ Tuple[str, FunctionNode, str, ScopeAnalyzer, Scope, Optional[str], Optional[str], List[str]]
109
+ ],
110
+ class_infos: List[ClassInfo],
111
+ ) -> None:
112
+ """Initializer that primes each worker process with engine state and function context."""
113
+
114
+ global _worker_engine, _worker_functions, _worker_class_infos
115
+
116
+ max_params = engine_config["max_parameters"]
117
+ min_ln = engine_config["min_lines"]
118
+ _worker_engine = UnificationRefactorEngine(
119
+ max_parameters=int(max_params) if max_params is not None else DEFAULT_MAX_PARAMETERS,
120
+ min_lines=int(min_ln) if min_ln is not None else DEFAULT_MIN_LINES,
121
+ parameterize_constants=bool(engine_config["parameterize_constants"]),
122
+ prefer_absolute_imports=engine_config.get("prefer_absolute_imports"),
123
+ pep420_namespace_packages=engine_config.get("pep420_namespace_packages"),
124
+ )
125
+ _worker_functions = functions
126
+ _worker_class_infos = class_infos
127
+
128
+
129
+ def _process_pair_in_worker(
130
+ task: Tuple[int, CodeBlockPair],
131
+ ) -> Tuple[int, Optional[RefactoringProposal]]:
132
+ """Worker entry point that evaluates a single code block pair."""
133
+
134
+ if _worker_engine is None or _worker_functions is None or _worker_class_infos is None:
135
+ raise RuntimeError("Worker not initialized for pair processing")
136
+
137
+ pair_index, pair = task
138
+ proposal = _worker_engine._try_refactor_pair_multi_file(
139
+ pair, _worker_functions, _worker_class_infos
140
+ )
141
+ return pair_index, proposal
142
+
143
+
144
+ class UnificationRefactorEngine:
145
+ """
146
+ Main engine for unification-based refactoring.
147
+
148
+ This finds and extracts duplicate code using unification.
149
+ """
150
+
151
+ def __init__(
152
+ self,
153
+ max_parameters: int = DEFAULT_MAX_PARAMETERS,
154
+ min_lines: int = DEFAULT_MIN_LINES,
155
+ parameterize_constants: bool = True,
156
+ *,
157
+ prefer_absolute_imports: Optional[bool] = None,
158
+ pep420_namespace_packages: Optional[bool] = None,
159
+ promote_equal_hof_literals: bool = False,
160
+ ):
161
+ """
162
+ Initialize the refactoring engine.
163
+
164
+ Args:
165
+ max_parameters: Maximum parameters for extracted functions (default: 5)
166
+ min_lines: Minimum lines for a code block (default: 4)
167
+ parameterize_constants: Whether to parameterize differing constants
168
+ """
169
+ self.max_parameters = max_parameters
170
+ self.min_lines = min_lines
171
+ self.parameterize_constants = parameterize_constants
172
+ self.unifier = Unifier(
173
+ max_parameters=max_parameters,
174
+ parameterize_constants=parameterize_constants,
175
+ promote_equal_hof_literals=promote_equal_hof_literals,
176
+ )
177
+ self.extractor = HygienicExtractor()
178
+ # Cross-file import preferences
179
+ self.prefer_absolute_imports = prefer_absolute_imports
180
+ self.pep420_namespace_packages = pep420_namespace_packages
181
+ # Default behavior: allow safe handling of globals/nonlocals by not parameterizing
182
+ # them and promoting necessary declarations into the extracted function when needed.
183
+
184
+ # Memoization caches keyed by the identity of AST nodes parsed for this engine run.
185
+ self._assignment_cache: WeakKeyDictionary[ast.AST, Dict[int, bool]] = WeakKeyDictionary()
186
+ self._used_names_cache: WeakKeyDictionary[ast.AST, FrozenSet[str]] = WeakKeyDictionary()
187
+ # Track helper name allocation per canonical file so helpers remain unique.
188
+ self._helper_name_counters: Dict[str, int] = {}
189
+
190
+ # --- Debug helpers ---
191
+ def _debug_reject(
192
+ self, reason: str, pair: "CodeBlockPair", detail: Optional[str] = None
193
+ ) -> None:
194
+ """Emit a concise rejection line when DEBUG_PROPOSAL_REJECTIONS is set.
195
+
196
+ Includes function names and basic block ranges to help triage pruning gates.
197
+ """
198
+ try:
199
+ import os as _os
200
+
201
+ if not _os.getenv("DEBUG_PROPOSAL_REJECTIONS"):
202
+ return
203
+ msg = (
204
+ f"REJECT[{reason}]: {pair.function1_name}{'@'+str(pair.block1_range) if pair.block1_range else ''} "
205
+ f"<-> {pair.function2_name}{'@'+str(pair.block2_range) if pair.block2_range else ''}"
206
+ )
207
+ if detail:
208
+ msg += f" :: {detail}"
209
+ print(msg)
210
+ except Exception:
211
+ # Never let debug logging interfere with refactoring
212
+ pass
213
+
214
+ def analyze_file(self, file_path: str) -> List[RefactoringProposal]:
215
+ """
216
+ Analyze a Python file and find refactoring opportunities.
217
+
218
+ Args:
219
+ file_path: Path to Python file
220
+
221
+ Returns:
222
+ List of refactoring proposals
223
+ """
224
+ return self.analyze_files([file_path])
225
+
226
+ def analyze_directory(
227
+ self,
228
+ directory: str,
229
+ recursive: bool = True,
230
+ *,
231
+ verbose: bool = False,
232
+ progress: str = "tqdm",
233
+ ) -> List[RefactoringProposal]:
234
+ """
235
+ Analyze all Python files in a directory and find refactoring opportunities.
236
+
237
+ Args:
238
+ directory: Path to directory
239
+ recursive: Whether to search subdirectories (default: True)
240
+
241
+ Returns:
242
+ List of refactoring proposals
243
+ """
244
+ # Find all Python files
245
+ python_files = self._find_python_files(directory, recursive)
246
+
247
+ if not python_files:
248
+ return []
249
+
250
+ if verbose:
251
+ print(f"Found {len(python_files)} Python files in {directory}")
252
+
253
+ # Analyze all files together
254
+ return self.analyze_files(python_files, verbose=verbose, progress=progress)
255
+
256
+ def _find_python_files(self, directory: str, recursive: bool = True) -> List[str]:
257
+ """
258
+ Find all Python files in a directory.
259
+
260
+ Args:
261
+ directory: Directory to search
262
+ recursive: Whether to search subdirectories
263
+
264
+ Returns:
265
+ List of Python file paths
266
+ """
267
+ python_files = []
268
+ directory_path = Path(directory)
269
+
270
+ if not directory_path.exists():
271
+ return []
272
+
273
+ if recursive:
274
+ # Recursively find all .py files
275
+ for py_file in directory_path.rglob("*.py"):
276
+ # Skip common directories to ignore
277
+ if any(
278
+ part.startswith(".") or part in ["__pycache__", "venv", "env", "node_modules"]
279
+ for part in py_file.parts
280
+ ):
281
+ continue
282
+ python_files.append(str(py_file))
283
+ else:
284
+ # Only find .py files in this directory
285
+ for py_file in directory_path.glob("*.py"):
286
+ python_files.append(str(py_file))
287
+
288
+ return sorted(python_files)
289
+
290
+ def _get_assignment_reuse(self, func: FunctionNode) -> Dict[int, bool]:
291
+ """Return (and cache) assignment analysis for a function definition."""
292
+ cached = self._assignment_cache.get(func)
293
+ if cached is not None:
294
+ return cached
295
+ analysis = analyze_assignments(func)
296
+ self._assignment_cache[func] = analysis
297
+ return analysis
298
+
299
+ def analyze_files(
300
+ self,
301
+ file_paths: List[str],
302
+ *,
303
+ verbose: bool = False,
304
+ progress: str = "tqdm",
305
+ invalidate_paths: Optional[List[str]] = None,
306
+ ) -> List[RefactoringProposal]:
307
+ """Analyze multiple files using the compiler-style pipeline and return proposals.
308
+
309
+ invalidate_paths: If provided, forces reparse/reanalysis of these paths even if cached.
310
+ """
311
+ # Delegate to the pipeline for analysis while preserving existing behavior
312
+ return run_pipeline(
313
+ file_paths,
314
+ engine=self,
315
+ verbose=verbose,
316
+ progress=progress,
317
+ invalidate_paths=invalidate_paths,
318
+ )
319
+
320
+ def _process_block_pairs(
321
+ self,
322
+ block_pairs: List[CodeBlockPair],
323
+ all_functions: List[
324
+ Tuple[
325
+ str,
326
+ FunctionNode,
327
+ str,
328
+ ScopeAnalyzer,
329
+ Scope,
330
+ Optional[str],
331
+ Optional[str],
332
+ List[str],
333
+ ]
334
+ ],
335
+ class_infos: List[ClassInfo],
336
+ *,
337
+ verbose: bool,
338
+ progress: str,
339
+ ) -> List[RefactoringProposal]:
340
+ if not block_pairs:
341
+ return []
342
+
343
+ if self._should_use_parallel(len(block_pairs)):
344
+ try:
345
+ return self._evaluate_pairs_parallel(
346
+ block_pairs,
347
+ all_functions,
348
+ class_infos,
349
+ verbose=verbose,
350
+ progress=progress,
351
+ )
352
+ except Exception:
353
+ if verbose:
354
+ print("Parallel pair evaluation failed; falling back to serial execution")
355
+ # Fall back to serial evaluation if multiprocessing encounters an issue
356
+ return self._evaluate_pairs_serial(
357
+ block_pairs,
358
+ all_functions,
359
+ class_infos,
360
+ verbose=verbose,
361
+ progress=progress,
362
+ )
363
+
364
+ return self._evaluate_pairs_serial(
365
+ block_pairs,
366
+ all_functions,
367
+ class_infos,
368
+ verbose=verbose,
369
+ progress=progress,
370
+ )
371
+
372
+ def _should_use_parallel(self, pair_count: int) -> bool:
373
+ """Decide if multiprocessing should be used for pair evaluation.
374
+
375
+ Benchmarks run in Nov 2025 showed the ProcessPoolExecutor path to be
376
+ roughly 0.6× slower than the serial evaluator because the work per
377
+ candidate is too small to amortize process start and AST pickling cost.
378
+ We keep this guard so future batching/tuning work can flip the switch
379
+ without reworking the call site, but for now we always stay serial.
380
+ """
381
+ return False
382
+
383
+ @staticmethod
384
+ def _load_tqdm_wrapper() -> Optional[Any]:
385
+ """Best-effort tqdm importer (mirrors DRY run helper, see docs/DRY_RUN_2025-11-28.md)."""
386
+
387
+ try:
388
+ import importlib
389
+
390
+ module = importlib.import_module("tqdm.auto")
391
+ return getattr(module, "tqdm")
392
+ except Exception:
393
+ return None
394
+
395
+ @staticmethod
396
+ def _start_inline_status(label: str, enabled: bool) -> None:
397
+ """Emit the leading inline progress label when requested (DRY helper)."""
398
+
399
+ if enabled:
400
+ print(label, end=" ", flush=True)
401
+
402
+ @staticmethod
403
+ def _render_inline_bar(pct: int, bar_len: int = 24) -> str:
404
+ """Render a textual progress bar reused across inline progress sites."""
405
+
406
+ filled = (pct * bar_len) // 100
407
+ return "#" * filled + "-" * (bar_len - filled)
408
+
409
+ @classmethod
410
+ def _update_inline_status(
411
+ cls, label: str, pct: int, *, bar_len: int = 24, suffix: str = ""
412
+ ) -> None:
413
+ """Print an inline progress update with consistent formatting."""
414
+
415
+ bar = cls._render_inline_bar(pct, bar_len=bar_len)
416
+ suffix_text = f" {suffix}" if suffix else ""
417
+ print(f"\r{label} [{bar}] {pct:3d}%{suffix_text}", end="", flush=True)
418
+
419
+ @staticmethod
420
+ def _finish_inline_status(enabled: bool) -> None:
421
+ """Terminate the inline status line so subsequent logs stay readable."""
422
+
423
+ if enabled:
424
+ print()
425
+
426
+ @staticmethod
427
+ def _pop_next_proposal(queue: List[RefactoringProposal]) -> Optional[RefactoringProposal]:
428
+ """Remove and return the oldest queued proposal (DRY helper; see docs/DRY_RUN_2025-11-28.md)."""
429
+
430
+ if not queue:
431
+ return None
432
+ return queue.pop(0)
433
+
434
+ def _resolve_progress_backend(self, progress: str) -> Tuple[str, Optional[Any], bool]:
435
+ """Normalize progress flag and load tqdm when available (mirrors DRY helper guidance)."""
436
+
437
+ allowed = {"auto", "tqdm", "none", "detail"}
438
+ normalized = progress if progress in allowed else "tqdm"
439
+ use_tqdm = normalized in {"auto", "tqdm"}
440
+ tqdm_cls: Optional[Any] = None
441
+ if use_tqdm:
442
+ tqdm_cls = self._load_tqdm_wrapper()
443
+ use_tqdm = tqdm_cls is not None
444
+ return normalized, tqdm_cls, use_tqdm
445
+
446
+ @staticmethod
447
+ def _function_contains_nonlocal(func: FunctionNode) -> bool:
448
+ """Return True if the function body contains any nonlocal declarations."""
449
+
450
+ for stmt in func.body:
451
+ # Skip nested definitions; only consider nonlocal statements that belong
452
+ # to the function itself. Nonlocals inside nested functions do not impact
453
+ # whether the outer function may be safely extracted.
454
+ if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
455
+ continue
456
+ for node in ast.walk(stmt):
457
+ if isinstance(node, ast.Nonlocal):
458
+ return True
459
+ return False
460
+
461
+ @staticmethod
462
+ def _decorator_name(decorator: ast.expr) -> Optional[str]:
463
+ """Return the simple name for a decorator expression if it can be resolved."""
464
+
465
+ if isinstance(decorator, ast.Name):
466
+ return decorator.id
467
+ if isinstance(decorator, ast.Attribute):
468
+ return decorator.attr
469
+ if isinstance(decorator, ast.Call):
470
+ return UnificationRefactorEngine._decorator_name(decorator.func)
471
+ return None
472
+
473
+ @staticmethod
474
+ def _has_decorator(fn: ast.FunctionDef, name: str) -> bool:
475
+ """Return True when the function already carries a decorator with the given name."""
476
+
477
+ return any(
478
+ UnificationRefactorEngine._decorator_name(dec) == name for dec in fn.decorator_list
479
+ )
480
+
481
+ @staticmethod
482
+ def _strip_decorator(fn: ast.FunctionDef, name: str) -> None:
483
+ """Remove any decorator whose resolved name matches ``name``."""
484
+
485
+ fn.decorator_list = [
486
+ dec
487
+ for dec in fn.decorator_list
488
+ if UnificationRefactorEngine._decorator_name(dec) != name
489
+ ]
490
+
491
+ @staticmethod
492
+ def _ensure_leading_param(fn: ast.FunctionDef, param_name: str) -> None:
493
+ """Ensure the positional-args list starts with ``param_name`` (preserving annotations)."""
494
+
495
+ existing: Optional[ast.arg] = None
496
+ remaining: List[ast.arg] = []
497
+ for arg in fn.args.args:
498
+ if arg.arg == param_name and existing is None:
499
+ existing = arg
500
+ continue
501
+ if arg.arg == param_name:
502
+ # Drop duplicate occurrences beyond the first
503
+ continue
504
+ remaining.append(arg)
505
+
506
+ if existing is None:
507
+ existing = ast.arg(arg=param_name)
508
+
509
+ fn.args.args = [existing] + remaining
510
+
511
+ @staticmethod
512
+ def _retarget_helper_calls(node: ast.AST, original_name: str, final_name: str) -> ast.AST:
513
+ """Rewrite helper call-sites when the extracted helper is renamed (DRY helper parity)."""
514
+
515
+ if original_name == final_name:
516
+ return node
517
+
518
+ class _CallRenamer(ast.NodeTransformer):
519
+ def __init__(self, old: str, new: str) -> None:
520
+ self.old = old
521
+ self.new = new
522
+
523
+ def visit_Call(
524
+ self, call: ast.Call
525
+ ) -> ast.AST: # pragma: no cover - simple AST rewrite
526
+ updated = cast(ast.Call, self.generic_visit(call))
527
+ if isinstance(updated.func, ast.Name) and updated.func.id == self.old:
528
+ updated.func.id = self.new
529
+ return updated
530
+
531
+ return _CallRenamer(original_name, final_name).visit(node)
532
+
533
+ @staticmethod
534
+ def _scan_module_docstring_and_imports(lines: List[str]) -> Tuple[int, int]:
535
+ """Return last import line and docstring boundary (DRY helper parity)."""
536
+
537
+ in_docstring = False
538
+ docstring_char: Optional[str] = None
539
+ last_import_line = 0
540
+ after_docstring = 0
541
+
542
+ for i, line in enumerate(lines):
543
+ stripped = line.strip()
544
+
545
+ if i == 0 and (stripped.startswith('"""') or stripped.startswith("'''")):
546
+ docstring_char = stripped[:3]
547
+ if stripped.count(docstring_char) < 2:
548
+ in_docstring = True
549
+ else:
550
+ after_docstring = i + 1
551
+ continue
552
+
553
+ if in_docstring:
554
+ assert docstring_char is not None
555
+ if docstring_char in stripped:
556
+ in_docstring = False
557
+ after_docstring = i + 1
558
+ continue
559
+
560
+ if stripped.startswith("import ") or stripped.startswith("from "):
561
+ last_import_line = i + 1
562
+ continue
563
+
564
+ if last_import_line > 0 and stripped and not stripped.startswith("#"):
565
+ break
566
+
567
+ return last_import_line, after_docstring
568
+
569
+ def _allocate_helper_name(self, file_path: str) -> str:
570
+ """Return a unique helper name for the given canonical file."""
571
+
572
+ counter = self._helper_name_counters.get(file_path)
573
+ if counter is None:
574
+ counter = self._discover_helper_counter_seed(file_path)
575
+ helper_name = f"__extracted_func_{counter}"
576
+ self._helper_name_counters[file_path] = counter + 1
577
+ return helper_name
578
+
579
+ def _discover_helper_counter_seed(self, file_path: str) -> int:
580
+ """Prime the helper counter based on existing helper names in a file."""
581
+
582
+ try:
583
+ content = Path(file_path).read_text(encoding="utf-8")
584
+ except Exception:
585
+ return 0
586
+
587
+ pattern = re.compile(r"__extracted_func(?:_(\d+))?")
588
+ max_seen = -1
589
+ for match in pattern.finditer(content):
590
+ suffix = match.group(1)
591
+ idx = int(suffix) if suffix is not None else 0
592
+ if idx > max_seen:
593
+ max_seen = idx
594
+ return max_seen + 1
595
+
596
+ def _prepare_extracted_method_signature(
597
+ self,
598
+ fn: ast.FunctionDef,
599
+ method_kind: Literal["instance", "classmethod", "staticmethod"],
600
+ implicit_param: Optional[str],
601
+ ) -> None:
602
+ """Normalize the extracted helper so it behaves like the requested method type."""
603
+
604
+ if method_kind == "instance":
605
+ name = implicit_param or "self"
606
+ self._ensure_leading_param(fn, name)
607
+ # Strip any conflicting decorators that might have been synthesized earlier
608
+ self._strip_decorator(fn, "staticmethod")
609
+ self._strip_decorator(fn, "classmethod")
610
+ elif method_kind == "classmethod":
611
+ name = implicit_param or "cls"
612
+ self._ensure_leading_param(fn, name)
613
+ self._strip_decorator(fn, "staticmethod")
614
+ if not self._has_decorator(fn, "classmethod"):
615
+ fn.decorator_list.insert(0, ast.Name(id="classmethod", ctx=ast.Load()))
616
+ elif method_kind == "staticmethod":
617
+ self._strip_decorator(fn, "classmethod")
618
+ if not self._has_decorator(fn, "staticmethod"):
619
+ fn.decorator_list.insert(0, ast.Name(id="staticmethod", ctx=ast.Load()))
620
+ else:
621
+ raise ValueError(f"Unsupported method kind: {method_kind}")
622
+
623
+ def _rewrite_call_for_method(
624
+ self,
625
+ node: ast.AST,
626
+ original_name: str,
627
+ new_name: str,
628
+ method_kind: Optional[Literal["instance", "classmethod", "staticmethod"]],
629
+ implicit_param: Optional[str],
630
+ class_name: Optional[str],
631
+ ) -> ast.AST:
632
+ """Rewrite calls to the extracted helper so they use method dispatch semantics."""
633
+
634
+ if method_kind is None:
635
+ return node
636
+
637
+ rewriter = MethodCallRewriter(
638
+ self._drop_implicit_positional,
639
+ self._drop_implicit_keyword,
640
+ original_name=original_name,
641
+ new_name=new_name,
642
+ method_kind=method_kind,
643
+ implicit_name=implicit_param,
644
+ class_name=class_name,
645
+ )
646
+ return rewriter.visit(node) # type: ignore[no-any-return]
647
+
648
+ @staticmethod
649
+ def _drop_implicit_positional(args: List[ast.expr], implicit_name: str) -> List[ast.expr]:
650
+ """Drop the first positional argument matching ``implicit_name`` if present."""
651
+
652
+ result: List[ast.expr] = []
653
+ dropped = False
654
+ for arg in args:
655
+ if not dropped and isinstance(arg, ast.Name) and arg.id == implicit_name:
656
+ dropped = True
657
+ continue
658
+ result.append(arg)
659
+ return result
660
+
661
+ @staticmethod
662
+ def _drop_implicit_keyword(
663
+ keywords: List[ast.keyword], implicit_name: str
664
+ ) -> List[ast.keyword]:
665
+ """Drop the first keyword argument whose name matches ``implicit_name``."""
666
+
667
+ result: List[ast.keyword] = []
668
+ dropped = False
669
+ for kw in keywords:
670
+ if not dropped and kw.arg == implicit_name:
671
+ dropped = True
672
+ continue
673
+ result.append(kw)
674
+ return result
675
+
676
+ def _get_method_context(
677
+ self, func: Optional[FunctionNode], class_name: Optional[str]
678
+ ) -> MethodInfo:
679
+ """Return method metadata for ``func`` when it is defined inside ``class_name``."""
680
+
681
+ if func is None or class_name is None:
682
+ return MethodInfo(kind=None, implicit_param=None)
683
+
684
+ kind: Optional[Literal["instance", "classmethod", "staticmethod"]] = None
685
+ for decorator in func.decorator_list:
686
+ name = self._decorator_name(decorator)
687
+ if name == "staticmethod":
688
+ kind = "staticmethod"
689
+ break
690
+ if name == "classmethod":
691
+ kind = "classmethod"
692
+ break
693
+
694
+ if kind is None:
695
+ kind = "instance"
696
+
697
+ implicit_param = None
698
+ if kind in {"instance", "classmethod"}:
699
+ if func.args.args:
700
+ implicit_param = func.args.args[0].arg
701
+ else:
702
+ implicit_param = "self" if kind == "instance" else "cls"
703
+
704
+ return MethodInfo(kind=kind, implicit_param=implicit_param)
705
+
706
+ @staticmethod
707
+ def _resolve_base_name(expr: ast.expr) -> Optional[str]:
708
+ """Resolve a base-class expression into its dotted name when feasible."""
709
+
710
+ if isinstance(expr, ast.Name):
711
+ return expr.id
712
+ if isinstance(expr, ast.Attribute):
713
+ parts: List[str] = []
714
+ current: ast.expr = expr
715
+ while isinstance(current, ast.Attribute):
716
+ parts.append(current.attr)
717
+ current = current.value
718
+ if isinstance(current, ast.Name):
719
+ parts.append(current.id)
720
+ return ".".join(reversed(parts))
721
+ return None
722
+
723
+ @staticmethod
724
+ def _class_info_key(info: ClassInfo) -> Tuple[str, str]:
725
+ """Return a stable identifier for a class definition."""
726
+
727
+ return (info.file_path, info.qualname)
728
+
729
+ def _find_class_info_by_name(
730
+ self, class_infos: List[ClassInfo], file_path: str, class_name: str
731
+ ) -> Optional[ClassInfo]:
732
+ """Locate class metadata using its defining file and simple name."""
733
+
734
+ matches = [
735
+ info for info in class_infos if info.file_path == file_path and info.name == class_name
736
+ ]
737
+ if not matches:
738
+ return None
739
+ if len(matches) == 1:
740
+ return matches[0]
741
+
742
+ # Prefer the innermost definition (longest qualname) when duplicates exist.
743
+ matches.sort(key=lambda info: info.qualname.count("."), reverse=True)
744
+ return matches[0]
745
+
746
+ def _find_class_info_for_base(
747
+ self,
748
+ class_infos: List[ClassInfo],
749
+ base_name: str,
750
+ *,
751
+ prefer_file: Optional[str] = None,
752
+ ) -> Optional[ClassInfo]:
753
+ """Resolve a base-class reference to known class metadata when possible."""
754
+
755
+ # Exact qualname match first (covers nested classes written as Outer.Inner)
756
+ qual_matches = [info for info in class_infos if info.qualname == base_name]
757
+ if prefer_file is not None:
758
+ for info in qual_matches:
759
+ if info.file_path == prefer_file:
760
+ return info
761
+ if qual_matches:
762
+ return qual_matches[0]
763
+
764
+ simple_name = base_name.split(".")[-1]
765
+ simple_matches = [info for info in class_infos if info.name == simple_name]
766
+ if prefer_file is not None:
767
+ for info in simple_matches:
768
+ if info.file_path == prefer_file:
769
+ return info
770
+ if len(simple_matches) == 1:
771
+ return simple_matches[0]
772
+ return None
773
+
774
+ def _collect_class_ancestors(
775
+ self, class_info: ClassInfo, class_infos: List[ClassInfo]
776
+ ) -> List[ClassInfo]:
777
+ """Return ancestors starting from the nearest base class."""
778
+
779
+ ancestors: List[Tuple[int, ClassInfo]] = []
780
+ visited: Set[Tuple[str, str]] = set()
781
+ queue: deque[Tuple[ClassInfo, int]] = deque([(class_info, 0)])
782
+
783
+ while queue:
784
+ current, depth = queue.popleft()
785
+ for base_name in current.bases:
786
+ base_info = self._find_class_info_for_base(
787
+ class_infos, base_name, prefer_file=current.file_path
788
+ )
789
+ if base_info is None:
790
+ continue
791
+ key = self._class_info_key(base_info)
792
+ if key in visited:
793
+ continue
794
+ visited.add(key)
795
+ ancestors.append((depth + 1, base_info))
796
+ queue.append((base_info, depth + 1))
797
+
798
+ ancestors.sort(key=lambda item: item[0])
799
+ return [info for _depth, info in ancestors]
800
+
801
+ def _find_common_ancestor(
802
+ self,
803
+ class1: Tuple[str, str],
804
+ class2: Tuple[str, str],
805
+ class_infos: List[ClassInfo],
806
+ ) -> Optional[ClassInfo]:
807
+ """Return the nearest shared ancestor class for two class definitions."""
808
+
809
+ file1, name1 = class1
810
+ file2, name2 = class2
811
+
812
+ info1 = self._find_class_info_by_name(class_infos, file1, name1)
813
+ info2 = self._find_class_info_by_name(class_infos, file2, name2)
814
+ if info1 is None or info2 is None:
815
+ return None
816
+
817
+ key1 = self._class_info_key(info1)
818
+ key2 = self._class_info_key(info2)
819
+
820
+ chain1 = [info1] + self._collect_class_ancestors(info1, class_infos)
821
+ chain2 = [info2] + self._collect_class_ancestors(info2, class_infos)
822
+ lookup2 = {self._class_info_key(info): info for info in chain2}
823
+
824
+ for info in chain1:
825
+ key = self._class_info_key(info)
826
+ if key in lookup2:
827
+ if key == key1 and key == key2:
828
+ # Identical class; handled elsewhere.
829
+ continue
830
+ return lookup2[key]
831
+ return None
832
+
833
+ def _choose_class_insertion(
834
+ self,
835
+ pair: CodeBlockPair,
836
+ method_info1: MethodInfo,
837
+ method_info2: MethodInfo,
838
+ class_infos: List[ClassInfo],
839
+ ) -> Optional[ClassInsertionPlan]:
840
+ """Determine whether the helper should be inserted into a class context."""
841
+ # Original logic required both method kinds to be equal and non-None. In practice
842
+ # some methods may not have their kind inferred (kind=None) even though they are
843
+ # instance methods (implicit_param present / first arg named 'self'). We relax
844
+ # this by attempting inference when BOTH kinds are None; if only one side is
845
+ # None we keep the conservative skip to avoid mismatches (e.g., staticmethod
846
+ # vs instance).
847
+ k1, k2 = method_info1.kind, method_info2.kind
848
+ inferred_kind: Optional[Literal["instance", "classmethod", "staticmethod"]] = None
849
+ if k1 is None and k2 is None:
850
+ # Infer 'instance' if both have an implicit_param or a first argument name.
851
+ # method_info.implicit_param is populated for recognized methods; as a
852
+ # fallback, treat absence uniformly as instance.
853
+ inferred_kind = "instance"
854
+ else:
855
+ if k1 != k2 or k1 is None or k2 is None:
856
+ return None
857
+ if pair.class1_name is None or pair.class2_name is None:
858
+ return None
859
+
860
+ file1 = pair.file_path
861
+ file2 = pair.file_path2 or pair.file_path
862
+
863
+ # At this point we know effective_kind is valid because we've already validated k1/k2
864
+ effective_kind: Literal["instance", "classmethod", "staticmethod"] = cast(
865
+ Literal["instance", "classmethod", "staticmethod"], inferred_kind or method_info1.kind
866
+ )
867
+ if file1 == file2 and pair.class1_name == pair.class2_name:
868
+ implicit_param = method_info1.implicit_param or method_info2.implicit_param
869
+ if effective_kind == "instance" and not implicit_param:
870
+ implicit_param = "self"
871
+ if effective_kind == "classmethod" and not implicit_param:
872
+ implicit_param = "cls"
873
+ return ClassInsertionPlan(
874
+ class_name=pair.class1_name,
875
+ file_path=file1,
876
+ method_kind=effective_kind,
877
+ implicit_param=implicit_param,
878
+ )
879
+
880
+ ancestor = self._find_common_ancestor(
881
+ (file1, pair.class1_name),
882
+ (file2, pair.class2_name),
883
+ class_infos,
884
+ )
885
+ if ancestor is None:
886
+ return None
887
+
888
+ implicit_param = method_info1.implicit_param or method_info2.implicit_param
889
+ if effective_kind == "instance" and not implicit_param:
890
+ implicit_param = "self"
891
+ if effective_kind == "classmethod" and not implicit_param:
892
+ implicit_param = "cls"
893
+
894
+ return ClassInsertionPlan(
895
+ class_name=ancestor.name,
896
+ file_path=ancestor.file_path,
897
+ method_kind=effective_kind,
898
+ implicit_param=implicit_param,
899
+ )
900
+
901
+ def _evaluate_pairs_serial(
902
+ self,
903
+ block_pairs: List[CodeBlockPair],
904
+ all_functions: List[
905
+ Tuple[
906
+ str,
907
+ FunctionNode,
908
+ str,
909
+ ScopeAnalyzer,
910
+ Scope,
911
+ Optional[str],
912
+ Optional[str],
913
+ List[str],
914
+ ]
915
+ ],
916
+ class_infos: List[ClassInfo],
917
+ *,
918
+ verbose: bool,
919
+ progress: str,
920
+ ) -> List[RefactoringProposal]:
921
+ proposals: List[RefactoringProposal] = []
922
+
923
+ progress_mode, tqdm_cls, use_tqdm = self._resolve_progress_backend(progress)
924
+ tqdm_iter = None
925
+ # Always show progress for pair evaluation when progress is enabled, even if verbose=False
926
+ if use_tqdm and tqdm_cls is not None:
927
+ tqdm_iter = tqdm_cls(
928
+ block_pairs,
929
+ total=len(block_pairs),
930
+ desc="unify",
931
+ unit="pair",
932
+ leave=False,
933
+ )
934
+
935
+ if use_tqdm and tqdm_iter is not None:
936
+ for pair in tqdm_iter:
937
+ proposal = self._try_refactor_pair_multi_file(pair, all_functions, class_infos)
938
+ if proposal:
939
+ proposals.append(proposal)
940
+ return proposals
941
+
942
+ use_inline_bar = progress_mode in ("auto", "tqdm") and len(block_pairs) > 0 and not use_tqdm
943
+ last_pct = -1
944
+ self._start_inline_status("Analyzing pairs (unify):", use_inline_bar)
945
+
946
+ for idx, pair in enumerate(block_pairs, 1):
947
+ proposal = self._try_refactor_pair_multi_file(pair, all_functions, class_infos)
948
+ if proposal:
949
+ proposals.append(proposal)
950
+ if use_inline_bar:
951
+ pct = int(100 * idx / len(block_pairs))
952
+ if pct != last_pct:
953
+ last_pct = pct
954
+ self._update_inline_status("Analyzing pairs (unify):", pct)
955
+
956
+ self._finish_inline_status(use_inline_bar)
957
+
958
+ return proposals
959
+
960
+ def _evaluate_pairs_parallel(
961
+ self,
962
+ block_pairs: List[CodeBlockPair],
963
+ all_functions: List[
964
+ Tuple[
965
+ str,
966
+ FunctionNode,
967
+ str,
968
+ ScopeAnalyzer,
969
+ Scope,
970
+ Optional[str],
971
+ Optional[str],
972
+ List[str],
973
+ ]
974
+ ],
975
+ class_infos: List[ClassInfo],
976
+ *,
977
+ verbose: bool,
978
+ progress: str,
979
+ ) -> List[RefactoringProposal]:
980
+ progress_mode, tqdm_cls, tqdm_available = self._resolve_progress_backend(progress)
981
+
982
+ pair_count = len(block_pairs)
983
+ cpu_count = os.cpu_count() or 1
984
+ max_workers = min(cpu_count, pair_count)
985
+ if max_workers <= 1:
986
+ return self._evaluate_pairs_serial(
987
+ block_pairs,
988
+ all_functions,
989
+ class_infos,
990
+ verbose=verbose,
991
+ progress=progress,
992
+ )
993
+
994
+ engine_config: Dict[str, Optional[object]] = {
995
+ "max_parameters": self.max_parameters,
996
+ "min_lines": self.min_lines,
997
+ "parameterize_constants": self.parameterize_constants,
998
+ "prefer_absolute_imports": self.prefer_absolute_imports,
999
+ "pep420_namespace_packages": self.pep420_namespace_packages,
1000
+ }
1001
+
1002
+ tasks = [(idx, pair) for idx, pair in enumerate(block_pairs)]
1003
+ ordered_results: List[Optional[RefactoringProposal]] = [None] * pair_count
1004
+
1005
+ use_tqdm = verbose and tqdm_available and tqdm_cls is not None
1006
+ tqdm_wrapper = tqdm_cls if use_tqdm else None
1007
+
1008
+ use_inline_bar = verbose and not use_tqdm and progress_mode in ("auto", "tqdm")
1009
+ last_pct = -1
1010
+ completed = 0
1011
+ self._start_inline_status("Analyzing candidate pairs:", use_inline_bar)
1012
+
1013
+ # ProcessPoolExecutor type stubs are restrictive; cast to Any for initializer/initargs
1014
+ executor_kwargs: Dict[str, Any] = {
1015
+ "max_workers": max_workers,
1016
+ "initializer": _initialize_pair_worker,
1017
+ "initargs": (engine_config, all_functions, class_infos),
1018
+ }
1019
+ with ProcessPoolExecutor(**executor_kwargs) as executor:
1020
+ futures = [executor.submit(_process_pair_in_worker, task) for task in tasks]
1021
+ iterator = as_completed(futures)
1022
+ if use_tqdm and tqdm_wrapper is not None:
1023
+ iterator = tqdm_wrapper(
1024
+ iterator,
1025
+ total=pair_count,
1026
+ desc="Analyzing candidate pairs",
1027
+ unit="pair",
1028
+ leave=False,
1029
+ )
1030
+
1031
+ for future in iterator:
1032
+ pair_index, proposal = future.result()
1033
+ if proposal:
1034
+ ordered_results[pair_index] = proposal
1035
+ completed += 1
1036
+ if use_inline_bar:
1037
+ pct = int(100 * completed / pair_count)
1038
+ if pct != last_pct:
1039
+ last_pct = pct
1040
+ self._update_inline_status("Analyzing candidate pairs:", pct)
1041
+
1042
+ self._finish_inline_status(use_inline_bar)
1043
+
1044
+ return [proposal for proposal in ordered_results if proposal is not None]
1045
+
1046
+ @staticmethod
1047
+ def _block_line_span(block: Sequence[ast.stmt]) -> Optional[Tuple[int, int]]:
1048
+ """Return the (start_line, end_line) span for a contiguous block of statements."""
1049
+
1050
+ if not block:
1051
+ return None
1052
+
1053
+ start_node = block[0]
1054
+ end_node = block[-1]
1055
+
1056
+ start_line = getattr(start_node, "lineno", None)
1057
+ end_line = getattr(end_node, "end_lineno", None) or getattr(end_node, "lineno", None)
1058
+ if start_line is None or end_line is None:
1059
+ return None
1060
+
1061
+ return int(start_line), int(end_line)
1062
+
1063
+ @staticmethod
1064
+ def _body_without_docstring(body: Sequence[ast.stmt]) -> List[ast.stmt]:
1065
+ """Return body statements with a leading docstring removed when present."""
1066
+
1067
+ body_list = list(body)
1068
+ if not body_list:
1069
+ return []
1070
+
1071
+ first_stmt = body_list[0]
1072
+ if (
1073
+ isinstance(first_stmt, ast.Expr)
1074
+ and isinstance(first_stmt.value, ast.Constant)
1075
+ and isinstance(first_stmt.value.value, str)
1076
+ ):
1077
+ return body_list[1:]
1078
+
1079
+ return body_list
1080
+
1081
+ def _extract_code_blocks(
1082
+ self, function: FunctionNode
1083
+ ) -> List[Tuple[Tuple[int, int], List[ast.AST]]]:
1084
+ """
1085
+ Extract all contiguous code blocks from a function body, including nested bodies.
1086
+
1087
+ Args:
1088
+ function: Function definition
1089
+
1090
+ Returns:
1091
+ List of (line_range, statements) tuples
1092
+ """
1093
+
1094
+ def extract_from_body(body: List[ast.stmt]) -> List[Tuple[Tuple[int, int], List[ast.AST]]]:
1095
+ # Extract all contiguous subsequences of minimum length from a given body
1096
+ results: List[Tuple[Tuple[int, int], List[ast.AST]]] = []
1097
+
1098
+ # Extract all contiguous subsequences
1099
+ for length in range(len(body), 0, -1):
1100
+ for start in range(len(body) - length + 1):
1101
+ block = body[start : start + length]
1102
+
1103
+ # Do not extract blocks that contain nested function/class definitions.
1104
+ # These statements establish new scopes whose bindings must remain in the
1105
+ # original function so later statements can reference them.
1106
+ if any(
1107
+ isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))
1108
+ for stmt in block
1109
+ ):
1110
+ continue
1111
+
1112
+ span = self._block_line_span(block)
1113
+ if span is None:
1114
+ continue
1115
+ start_line, end_line = span
1116
+ line_count = end_line - start_line + 1
1117
+
1118
+ if line_count >= self.min_lines:
1119
+ results.append(((start_line, end_line), cast(List[ast.AST], block)))
1120
+
1121
+ # Recurse into nested bodies for control-flow/container statements
1122
+ for stmt in body:
1123
+ # Skip nested function/class definitions to avoid crossing scopes
1124
+ if isinstance(stmt, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
1125
+ continue
1126
+
1127
+ # Common body/orelse containers
1128
+ if hasattr(stmt, "body") and isinstance(getattr(stmt, "body"), list):
1129
+ results.extend(extract_from_body(getattr(stmt, "body")))
1130
+ if hasattr(stmt, "orelse") and isinstance(getattr(stmt, "orelse"), list):
1131
+ results.extend(extract_from_body(getattr(stmt, "orelse")))
1132
+
1133
+ # With and AsyncWith already covered by .body
1134
+ # Try/Except/Finally blocks
1135
+ if isinstance(stmt, ast.Try):
1136
+ if stmt.handlers:
1137
+ for h in stmt.handlers:
1138
+ if hasattr(h, "body") and isinstance(h.body, list):
1139
+ results.extend(extract_from_body(h.body))
1140
+ if hasattr(stmt, "finalbody") and isinstance(stmt.finalbody, list):
1141
+ results.extend(extract_from_body(stmt.finalbody))
1142
+
1143
+ return results
1144
+
1145
+ # Prepare top-level body (skip docstring)
1146
+ body = self._body_without_docstring(function.body)
1147
+
1148
+ return extract_from_body(body)
1149
+
1150
+ def _has_code_after_block(self, function: ast.FunctionDef, block_end_line: int) -> bool:
1151
+ """
1152
+ Check if there's any executable code after block_end_line in the function.
1153
+
1154
+ This is used to detect when extracting a block with returns would make
1155
+ subsequent code unreachable.
1156
+
1157
+ Args:
1158
+ function: Function definition
1159
+ block_end_line: End line of the block
1160
+
1161
+ Returns:
1162
+ True if there's code after the block
1163
+ """
1164
+ body = self._body_without_docstring(function.body)
1165
+
1166
+ # Check if any statement starts after block_end_line
1167
+ for stmt in body:
1168
+ if stmt.lineno > block_end_line:
1169
+ return True
1170
+ return False
1171
+
1172
+ def _has_returns_in_loops(self, block: List[ast.AST]) -> bool:
1173
+ """
1174
+ Check if a block contains return statements inside loops.
1175
+
1176
+ Returns in loops are conditional on the loop executing, so if the loop
1177
+ doesn't execute (e.g., empty iteration), control continues after the loop.
1178
+
1179
+ Args:
1180
+ block: List of AST statements
1181
+
1182
+ Returns:
1183
+ True if there are returns inside loop statements
1184
+ """
1185
+
1186
+ finder = LoopReturnFinder()
1187
+ for stmt in block:
1188
+ finder.visit(stmt)
1189
+ return finder.has_loop_return
1190
+
1191
+ def _get_used_names(self, node: ast.AST) -> Set[str]:
1192
+ """
1193
+ Get all variable names that are used (read from) in an AST node.
1194
+
1195
+ This collects all Name nodes with Load context.
1196
+
1197
+ Args:
1198
+ node: AST node to analyze
1199
+
1200
+ Returns:
1201
+ Set of variable names that are read in the node
1202
+ """
1203
+ cached = self._used_names_cache.get(node)
1204
+ if cached is not None:
1205
+ return set(cached)
1206
+
1207
+ collector = NameCollector()
1208
+ collector.visit(node)
1209
+
1210
+ frozen = frozenset(collector.used)
1211
+ self._used_names_cache[node] = frozen
1212
+ return set(frozen)
1213
+
1214
+ def _collect_parameter_names(self, func: FunctionNode) -> Set[str]:
1215
+ """Return all argument names for a function (including pos-only and varargs)."""
1216
+
1217
+ if not isinstance(func, (ast.FunctionDef, ast.AsyncFunctionDef)):
1218
+ return set()
1219
+
1220
+ params: Set[str] = set()
1221
+ args = func.args
1222
+
1223
+ for arg in getattr(args, "posonlyargs", []) or []:
1224
+ params.add(arg.arg)
1225
+ for arg in args.args:
1226
+ params.add(arg.arg)
1227
+ for arg in args.kwonlyargs:
1228
+ params.add(arg.arg)
1229
+ if args.vararg:
1230
+ params.add(args.vararg.arg)
1231
+ if args.kwarg:
1232
+ params.add(args.kwarg.arg)
1233
+
1234
+ return params
1235
+
1236
+ def _deepest_common_ancestry(
1237
+ self, anc1: Optional[List[str]], anc2: Optional[List[str]]
1238
+ ) -> Optional[str]:
1239
+ """Return deepest shared symbol in two ancestry chains (DRY helper \u00a7ref docs/DRY_RUN_2025-11-28.md)."""
1240
+
1241
+ if not anc1 or not anc2:
1242
+ return None
1243
+
1244
+ dce: Optional[str] = None
1245
+ for left, right in zip(anc1, anc2):
1246
+ if left == right:
1247
+ dce = left
1248
+ else:
1249
+ break
1250
+ return dce
1251
+
1252
+ def _build_block_binding_snapshot(
1253
+ self,
1254
+ func: FunctionNode,
1255
+ block_nodes: List[ast.AST],
1256
+ block_range: Tuple[int, int],
1257
+ reassignments: Dict[int, bool],
1258
+ ) -> BlockBindingSnapshot:
1259
+ """Aggregate binding stats for a block (DRY helper import, see docs/DRY_RUN_2025-11-28.md)."""
1260
+
1261
+ bound_in_block, reassigned_in_block = _collect_block_binding_stats(
1262
+ block_nodes, reassignments
1263
+ )
1264
+
1265
+ bound_before_block: Set[str] = set()
1266
+ block_start_line = block_range[0]
1267
+ for stmt in func.body:
1268
+ if hasattr(stmt, "lineno") and stmt.lineno < block_start_line:
1269
+ stmt_bound: Set[str] = set()
1270
+ stmt_reassigned: Set[str] = set()
1271
+ _collect_bindings_and_reassignments(
1272
+ stmt, reassignments, stmt_bound, stmt_reassigned
1273
+ )
1274
+ bound_before_block.update(stmt_bound)
1275
+
1276
+ bound_before_block.update(self._collect_parameter_names(func))
1277
+
1278
+ bound_after_block: Set[str] = set()
1279
+ block_end_line = block_range[1]
1280
+ for stmt in func.body:
1281
+ if hasattr(stmt, "lineno") and stmt.lineno > block_end_line:
1282
+ stmt_bound = set()
1283
+ stmt_reassigned = set()
1284
+ _collect_bindings_and_reassignments(
1285
+ stmt, reassignments, stmt_bound, stmt_reassigned
1286
+ )
1287
+ bound_after_block.update(stmt_bound)
1288
+
1289
+ initially_bound = bound_in_block - bound_before_block
1290
+
1291
+ return BlockBindingSnapshot(
1292
+ bound_in_block=bound_in_block,
1293
+ reassigned_in_block=reassigned_in_block,
1294
+ bound_before_block=bound_before_block,
1295
+ bound_after_block=bound_after_block,
1296
+ initially_bound=initially_bound,
1297
+ )
1298
+
1299
+ def _find_return_variables(
1300
+ self,
1301
+ func: FunctionNode,
1302
+ block_range: Tuple[int, int],
1303
+ initially_bound: Set[str],
1304
+ *,
1305
+ debug_label: Optional[str] = None,
1306
+ ) -> Set[str]:
1307
+ """
1308
+ Determine which newly-bound variables are read after the block (DRY helper parity).
1309
+ """
1310
+
1311
+ if not initially_bound:
1312
+ return set()
1313
+
1314
+ block_end_line = block_range[1]
1315
+ result: Set[str] = set()
1316
+ debug_enabled = bool(os.getenv("DEBUG_VALIDATION"))
1317
+
1318
+ for stmt in func.body:
1319
+ if not hasattr(stmt, "lineno") or stmt.lineno <= block_end_line:
1320
+ continue
1321
+
1322
+ uses = self._get_used_names(stmt)
1323
+ if debug_enabled and debug_label:
1324
+ print(
1325
+ f" {debug_label}: stmt@{stmt.lineno} ({stmt.__class__.__name__}) uses {uses}"
1326
+ )
1327
+
1328
+ overlap = uses & initially_bound
1329
+ if overlap:
1330
+ result.update(overlap)
1331
+ if debug_enabled and debug_label:
1332
+ print(
1333
+ f" RETURN NEEDED ({debug_label}): Variable(s) {overlap} will be returned from extracted function"
1334
+ )
1335
+
1336
+ if debug_enabled and debug_label and result:
1337
+ print(f"{debug_label} requires returning: {result}")
1338
+
1339
+ return result
1340
+
1341
+ def _find_block_pairs_multi_file(
1342
+ self,
1343
+ all_functions: List[
1344
+ Tuple[
1345
+ str,
1346
+ FunctionNode,
1347
+ str,
1348
+ ScopeAnalyzer,
1349
+ Scope,
1350
+ Optional[str],
1351
+ Optional[str],
1352
+ List[str],
1353
+ ]
1354
+ ],
1355
+ *,
1356
+ progress: str = "none",
1357
+ ) -> List[CodeBlockPair]:
1358
+ """
1359
+ Find all non-overlapping pairs of code blocks across multiple files.
1360
+
1361
+ Args:
1362
+ all_functions: List of (file_path, function, source, scope_analyzer, root_scope)
1363
+
1364
+ Returns:
1365
+ List of code block pairs
1366
+ """
1367
+ pairs = []
1368
+
1369
+ # Progress setup
1370
+ use_tqdm = progress in ("tqdm", "auto")
1371
+ tqdm_bar = None
1372
+ total_funcs = len(all_functions)
1373
+ total_func_pairs = (total_funcs * (total_funcs - 1)) // 2 if total_funcs > 1 else 0
1374
+ if use_tqdm and total_func_pairs > 0:
1375
+ tqdm_cls = self._load_tqdm_wrapper()
1376
+ if tqdm_cls is not None:
1377
+ tqdm_bar = tqdm_cls(
1378
+ total=total_func_pairs,
1379
+ desc="pairs",
1380
+ unit="fp",
1381
+ dynamic_ncols=True,
1382
+ leave=False,
1383
+ )
1384
+ else:
1385
+ use_tqdm = False
1386
+
1387
+ use_inline = (not use_tqdm) and progress in ("tqdm", "auto") and total_func_pairs > 0
1388
+ last_pct = -1
1389
+ self._start_inline_status("Pairing blocks:", use_inline)
1390
+
1391
+ func_pairs_done = 0
1392
+
1393
+ # For each pair of functions (including across files)
1394
+ for i, entry1 in enumerate(all_functions):
1395
+ # Backward compatibility: allow 5-tuples (no class context)
1396
+ if len(entry1) >= 8:
1397
+ file1, func1, source1, analyzer1, scope1, class1, encl1, anc1 = entry1
1398
+ elif len(entry1) == 7:
1399
+ file1, func1, source1, analyzer1, scope1, class1, encl1 = entry1
1400
+ anc1 = []
1401
+ else:
1402
+ file1, func1, source1, analyzer1, scope1 = entry1
1403
+ class1 = None
1404
+ encl1 = None
1405
+ anc1 = []
1406
+
1407
+ for entry2 in all_functions[i + 1 :]:
1408
+ if len(entry2) >= 8:
1409
+ file2, func2, source2, analyzer2, scope2, class2, encl2, anc2 = entry2
1410
+ elif len(entry2) == 7:
1411
+ file2, func2, source2, analyzer2, scope2, class2, encl2 = entry2
1412
+ anc2 = []
1413
+ else:
1414
+ file2, func2, source2, analyzer2, scope2 = entry2
1415
+ class2 = None
1416
+ encl2 = None
1417
+ anc2 = []
1418
+ # Extract all code blocks from each function
1419
+ blocks1 = [
1420
+ (block_range, block_nodes, extract_block_signature(block_nodes))
1421
+ for block_range, block_nodes in self._extract_code_blocks(func1)
1422
+ ]
1423
+ blocks2 = [
1424
+ (block_range, block_nodes, extract_block_signature(block_nodes))
1425
+ for block_range, block_nodes in self._extract_code_blocks(func2)
1426
+ ]
1427
+
1428
+ # Compare all pairs of blocks
1429
+ for block1_range, block1_nodes, sig1 in blocks1:
1430
+ for block2_range, block2_nodes, sig2 in blocks2:
1431
+ # Must have same number of statements for unification
1432
+ if len(block1_nodes) != len(block2_nodes):
1433
+ continue
1434
+
1435
+ # Check minimum size
1436
+ start1, end1 = block1_range
1437
+ start2, end2 = block2_range
1438
+ if (end1 - start1 + 1) < self.min_lines or (
1439
+ end2 - start2 + 1
1440
+ ) < self.min_lines:
1441
+ continue
1442
+
1443
+ if not quick_filter(sig1, sig2):
1444
+ continue
1445
+
1446
+ # Create pair with all necessary context
1447
+ pair = CodeBlockPair(
1448
+ file_path=file1,
1449
+ function1_name=func1.name,
1450
+ function2_name=func2.name,
1451
+ block1_range=block1_range,
1452
+ block2_range=block2_range,
1453
+ block1_nodes=block1_nodes,
1454
+ block2_nodes=block2_nodes,
1455
+ file_path2=file2,
1456
+ class1_name=class1,
1457
+ class2_name=class2,
1458
+ enclosing_function1_name=encl1,
1459
+ enclosing_function2_name=encl2,
1460
+ function1_ancestry=anc1,
1461
+ function2_ancestry=anc2,
1462
+ scope_analyzer1=analyzer1,
1463
+ scope_analyzer2=analyzer2,
1464
+ root_scope1=scope1,
1465
+ root_scope2=scope2,
1466
+ source1=source1,
1467
+ source2=source2,
1468
+ function1_node=func1,
1469
+ function2_node=func2,
1470
+ )
1471
+ pairs.append(pair)
1472
+
1473
+ # Update progress per function pair
1474
+ func_pairs_done += 1
1475
+ if tqdm_bar is not None:
1476
+ try:
1477
+ tqdm_bar.update(1)
1478
+ if func_pairs_done % 20 == 0 or func_pairs_done == total_func_pairs:
1479
+ tqdm_bar.set_postfix({"pairs": len(pairs)}, refresh=True)
1480
+ except Exception:
1481
+ pass
1482
+ elif use_inline:
1483
+ pct = int(100 * func_pairs_done / max(total_func_pairs, 1))
1484
+ if pct != last_pct:
1485
+ last_pct = pct
1486
+ self._update_inline_status(
1487
+ "Pairing blocks:",
1488
+ pct,
1489
+ suffix=f"| pairs={len(pairs)}",
1490
+ )
1491
+ if tqdm_bar is not None:
1492
+ try:
1493
+ tqdm_bar.close()
1494
+ except Exception:
1495
+ pass
1496
+ self._finish_inline_status(use_inline)
1497
+
1498
+ return pairs
1499
+
1500
+ def _try_refactor_pair_multi_file(
1501
+ self,
1502
+ pair: CodeBlockPair,
1503
+ all_functions: List[
1504
+ Tuple[
1505
+ str,
1506
+ FunctionNode,
1507
+ str,
1508
+ ScopeAnalyzer,
1509
+ Scope,
1510
+ Optional[str],
1511
+ Optional[str],
1512
+ List[str],
1513
+ ]
1514
+ ],
1515
+ class_infos: List[ClassInfo],
1516
+ ) -> Optional[RefactoringProposal]:
1517
+ """
1518
+ Try to refactor a pair of code blocks using unification (cross-file support).
1519
+
1520
+ Args:
1521
+ pair: Code block pair (may be cross-file)
1522
+ all_functions: All functions being analyzed
1523
+ class_infos: Metadata about classes discovered in analyzed files
1524
+
1525
+ Returns:
1526
+ Refactoring proposal or None
1527
+ """
1528
+ # DEBUG logging
1529
+ import os
1530
+
1531
+ if os.getenv("DEBUG_VALIDATION"):
1532
+ print("\n=== _try_refactor_pair_multi_file called ===")
1533
+ print(f"Functions: {pair.function1_name} and {pair.function2_name}")
1534
+ print(f"Block1 range: {pair.block1_range}")
1535
+ print(f"Block2 range: {pair.block2_range}")
1536
+
1537
+ # Resolve contextual analyzers and scopes from the aggregated function list
1538
+ func1: Optional[FunctionNode] = pair.function1_node
1539
+ func2: Optional[FunctionNode] = pair.function2_node
1540
+ scope_analyzer1: Optional[ScopeAnalyzer] = None
1541
+ scope_analyzer2: Optional[ScopeAnalyzer] = None
1542
+ root_scope1: Optional[Scope] = None
1543
+ root_scope2: Optional[Scope] = None
1544
+
1545
+ for entry in all_functions:
1546
+ (
1547
+ file_path,
1548
+ func,
1549
+ _source,
1550
+ analyzer,
1551
+ root_scope_entry,
1552
+ _class_name,
1553
+ _enclosing_func,
1554
+ _ancestry,
1555
+ ) = entry
1556
+ if func1 is None and file_path == pair.file_path and func.name == pair.function1_name:
1557
+ func1 = func
1558
+ scope_analyzer1 = analyzer
1559
+ root_scope1 = root_scope_entry
1560
+ if (
1561
+ func2 is None
1562
+ and file_path == (pair.file_path2 or pair.file_path)
1563
+ and func.name == pair.function2_name
1564
+ ):
1565
+ func2 = func
1566
+ scope_analyzer2 = analyzer
1567
+ root_scope2 = root_scope_entry
1568
+ if func1 is not None and func2 is not None:
1569
+ break
1570
+
1571
+ # Fallback to the pair-provided analyzers/scopes when discovery fails
1572
+ scope_analyzer = scope_analyzer1 or pair.scope_analyzer1
1573
+ root_scope = root_scope1 or pair.root_scope1
1574
+ if scope_analyzer2 is None and pair.scope_analyzer2 is not None:
1575
+ scope_analyzer2 = pair.scope_analyzer2
1576
+ if root_scope2 is None and pair.root_scope2 is not None:
1577
+ root_scope2 = pair.root_scope2
1578
+
1579
+ method_info1 = self._get_method_context(func1, pair.class1_name)
1580
+ method_info2 = self._get_method_context(func2, pair.class2_name)
1581
+
1582
+ # (Removed specialized full-body extraction fast-path; reverting to generic pairing logic.)
1583
+
1584
+ # DEBUG logging
1585
+ import os
1586
+
1587
+ if os.getenv("DEBUG_VALIDATION"):
1588
+ print("\n=== Finding Functions ===")
1589
+ print(f"Looking for: {pair.function1_name} and {pair.function2_name}")
1590
+ print(f"Found func1: {func1 is not None}")
1591
+ print(f"Found func2: {func2 is not None}")
1592
+
1593
+ # CRITICAL: Validate that blocks don't contain reassignments without initial bindings
1594
+ # Initialize return_variables tracking
1595
+ # This will be populated if we find variables that need to be returned
1596
+ return_variables_block1: Set[str] = set()
1597
+ return_variables_block2: Set[str] = set()
1598
+
1599
+ bound_in_block1: Set[str] = set()
1600
+ bound_before_block1: Set[str] = set()
1601
+ bound_after_block1: Set[str] = set()
1602
+ initially_bound1: Set[str] = set()
1603
+
1604
+ bound_in_block2: Set[str] = set()
1605
+ bound_before_block2: Set[str] = set()
1606
+ bound_after_block2: Set[str] = set()
1607
+ initially_bound2: Set[str] = set()
1608
+
1609
+ # This prevents extracting code like "result = result + 10" when "result = x * 2"
1610
+ # is outside the block. Such extractions are fundamentally unsound.
1611
+ if func1 and func2:
1612
+ # Analyze assignments in both functions
1613
+ reassignments1 = self._get_assignment_reuse(func1)
1614
+ reassignments2 = self._get_assignment_reuse(func2)
1615
+
1616
+ # Check if block1 contains reassignments without bindings
1617
+ has_unsafe1, problematic_vars1 = has_reassignments_without_bindings(
1618
+ func1, pair.block1_nodes, reassignments1
1619
+ )
1620
+ if has_unsafe1:
1621
+ self._debug_reject("unsafe_reassignment_block1", pair, str(problematic_vars1))
1622
+ return None
1623
+
1624
+ # Check if block2 contains reassignments without bindings
1625
+ has_unsafe2, problematic_vars2 = has_reassignments_without_bindings(
1626
+ func2, pair.block2_nodes, reassignments2
1627
+ )
1628
+ if has_unsafe2:
1629
+ self._debug_reject("unsafe_reassignment_block2", pair, str(problematic_vars2))
1630
+ return None
1631
+
1632
+ # Consolidated helper mirrors DRY's _refactor_engine_helper_2 (docs/DRY_RUN_2025-11-28.md)
1633
+ block1_snapshot = self._build_block_binding_snapshot(
1634
+ func1, pair.block1_nodes, pair.block1_range, reassignments1
1635
+ )
1636
+ block2_snapshot = self._build_block_binding_snapshot(
1637
+ func2, pair.block2_nodes, pair.block2_range, reassignments2
1638
+ )
1639
+
1640
+ bound_in_block1 = block1_snapshot.bound_in_block
1641
+ bound_before_block1 = block1_snapshot.bound_before_block
1642
+ bound_after_block1 = block1_snapshot.bound_after_block
1643
+ initially_bound1 = block1_snapshot.initially_bound
1644
+
1645
+ bound_in_block2 = block2_snapshot.bound_in_block
1646
+ bound_before_block2 = block2_snapshot.bound_before_block
1647
+ bound_after_block2 = block2_snapshot.bound_after_block
1648
+ initially_bound2 = block2_snapshot.initially_bound
1649
+
1650
+ debug_enabled = bool(os.getenv("DEBUG_VALIDATION"))
1651
+ if debug_enabled:
1652
+ print("\n=== Block1 Validation Debug ===")
1653
+ print(f"Function: {pair.function1_name}")
1654
+ print(f"Block lines: {pair.block1_range}")
1655
+ print(f"Bound in block: {bound_in_block1}")
1656
+ print(f"Bound before block: {bound_before_block1}")
1657
+ print(f"Newly bound in block: {initially_bound1}")
1658
+
1659
+ return_variables_block1 = self._find_return_variables(
1660
+ func1,
1661
+ pair.block1_range,
1662
+ initially_bound1,
1663
+ debug_label="Block1 Validation Debug" if debug_enabled else None,
1664
+ )
1665
+
1666
+ if debug_enabled:
1667
+ print("\n=== Block2 Validation Debug ===")
1668
+ print(f"Function: {pair.function2_name}")
1669
+ print(f"Block lines: {pair.block2_range}")
1670
+ print(f"Bound in block: {bound_in_block2}")
1671
+ print(f"Bound before block: {bound_before_block2}")
1672
+ print(f"Newly bound in block: {initially_bound2}")
1673
+
1674
+ return_variables_block2 = self._find_return_variables(
1675
+ func2,
1676
+ pair.block2_range,
1677
+ initially_bound2,
1678
+ debug_label="Block2 Validation Debug" if debug_enabled else None,
1679
+ )
1680
+
1681
+ # Check if both blocks are value-producing or both are not
1682
+ # Blocks with return_variables are treated as value-producing because
1683
+ # we will add return statements for those variables
1684
+ value_prod1 = is_value_producing(cast(List[ast.stmt], pair.block1_nodes)) or bool(
1685
+ return_variables_block1
1686
+ )
1687
+ value_prod2 = is_value_producing(cast(List[ast.stmt], pair.block2_nodes)) or bool(
1688
+ return_variables_block2
1689
+ )
1690
+
1691
+ if os.getenv("DEBUG_VALIDATION"):
1692
+ print(f" Value-producing check: block1={value_prod1}, block2={value_prod2}")
1693
+ if return_variables_block1:
1694
+ print(f" Block1 has return_variables: {return_variables_block1}")
1695
+ if return_variables_block2:
1696
+ print(f" Block2 has return_variables: {return_variables_block2}")
1697
+
1698
+ if value_prod1 != value_prod2:
1699
+ if os.getenv("DEBUG_VALIDATION"):
1700
+ print(" REJECTED: Value-producing mismatch")
1701
+ self._debug_reject("value_producing_mismatch", pair)
1702
+ return None
1703
+
1704
+ # CRITICAL: If blocks are NATURALLY value-producing (have return statements),
1705
+ # ensure complete return coverage. This prevents extracting partial control flow.
1706
+ # Skip this check for blocks that will have return statements ADDED for return_variables
1707
+ if value_prod1 and not return_variables_block1: # Naturally value-producing
1708
+ from .extractor import has_complete_return_coverage
1709
+
1710
+ if not has_complete_return_coverage(cast(List[ast.stmt], pair.block1_nodes)):
1711
+ if os.getenv("DEBUG_VALIDATION"):
1712
+ print(" REJECTED: Block1 missing complete return coverage")
1713
+ self._debug_reject("incomplete_return_coverage_block1", pair)
1714
+ return None
1715
+ if not has_complete_return_coverage(cast(List[ast.stmt], pair.block2_nodes)):
1716
+ if os.getenv("DEBUG_VALIDATION"):
1717
+ print(" REJECTED: Block2 missing complete return coverage")
1718
+ self._debug_reject("incomplete_return_coverage_block2", pair)
1719
+ return None
1720
+
1721
+ # Heuristic: avoid extracting trivial single-line return blocks that just
1722
+ # return a previously bound local name (e.g., `return result`). Prefer
1723
+ # extracting the preceding computation that produces the value.
1724
+ def _is_trivial_return_of_bound_name(
1725
+ block_nodes: List[ast.AST], bound_before_block: Set[str], bound_in_block: Set[str]
1726
+ ) -> bool:
1727
+ if len(block_nodes) != 1:
1728
+ return False
1729
+ stmt = block_nodes[0]
1730
+ return (
1731
+ isinstance(stmt, ast.Return)
1732
+ and isinstance(stmt.value, ast.Name)
1733
+ and stmt.value.id in bound_before_block
1734
+ and stmt.value.id not in bound_in_block
1735
+ )
1736
+
1737
+ if _is_trivial_return_of_bound_name(
1738
+ pair.block1_nodes, bound_before_block1, bound_in_block1
1739
+ ) and _is_trivial_return_of_bound_name(
1740
+ pair.block2_nodes, bound_before_block2, bound_in_block2
1741
+ ):
1742
+ if os.getenv("DEBUG_VALIDATION"):
1743
+ print(
1744
+ " REJECTED: Trivial single-line return blocks (prefer extracting computation)"
1745
+ )
1746
+ self._debug_reject("trivial_return_blocks", pair)
1747
+ return None
1748
+
1749
+ # Check structural similarity
1750
+ if not self._are_structurally_similar(pair.block1_nodes, pair.block2_nodes):
1751
+ if os.getenv("DEBUG_VALIDATION"):
1752
+ print(" REJECTED: Not structurally similar")
1753
+ self._debug_reject("not_structurally_similar", pair)
1754
+ return None
1755
+
1756
+ # Attempt unification
1757
+ blocks = [pair.block1_nodes, pair.block2_nodes]
1758
+ hygienic_renames: List[Dict[str, str]] = [{}, {}]
1759
+
1760
+ if os.getenv("DEBUG_VALIDATION"):
1761
+ print(" Attempting unification...")
1762
+
1763
+ try:
1764
+ substitution = self.unifier.unify_blocks(blocks, hygienic_renames)
1765
+ except Exception as e:
1766
+ if os.getenv("DEBUG_VALIDATION"):
1767
+ print(f" REJECTED: Unification exception: {e}")
1768
+ self._debug_reject("unification_exception", pair, str(e))
1769
+ return None
1770
+
1771
+ if not substitution:
1772
+ if os.getenv("DEBUG_VALIDATION"):
1773
+ print(" REJECTED: Unification failed (no substitution)")
1774
+ self._debug_reject("unification_failed", pair)
1775
+ return None
1776
+
1777
+ if os.getenv("DEBUG_VALIDATION"):
1778
+ print(" ✓ Unification successful")
1779
+ print(f" Substitution: {substitution}")
1780
+
1781
+ # Pre-compute the deepest common enclosing function (for same-file cases)
1782
+ dce_insert_func: Optional[str] = None
1783
+ try:
1784
+ same_file_ctx = pair.file_path2 is not None and pair.file_path2 == pair.file_path
1785
+ if same_file_ctx:
1786
+ dce_insert_func = self._deepest_common_ancestry(
1787
+ pair.function1_ancestry, pair.function2_ancestry
1788
+ )
1789
+ except Exception:
1790
+ dce_insert_func = None
1791
+
1792
+ # Get enclosing names to avoid shadowing (module-level by default)
1793
+ enclosing_names = set(root_scope.bindings.keys()) if root_scope else set()
1794
+ # If we plan to insert into a specific function scope (DCE), enrich hygiene set with that
1795
+ # function's local bindings to avoid name collisions
1796
+ if dce_insert_func:
1797
+ try:
1798
+ for fpath, fn, _src, analyzer, rscope, _cls, _encl, _anc in all_functions:
1799
+ if fpath == (pair.file_path2 or pair.file_path) and fn.name == dce_insert_func:
1800
+ func_scope = analyzer.node_scopes.get(fn)
1801
+ if func_scope is not None:
1802
+ enclosing_names.update(func_scope.bindings.keys())
1803
+ break
1804
+ except Exception:
1805
+ # Best effort only
1806
+ pass
1807
+ # Hygiene improvement: if we'll insert into a specific function scope (DCE),
1808
+ # include that function's local bindings to avoid name collisions.
1809
+ try:
1810
+ same_file_for_hygiene = (
1811
+ pair.file_path2 is not None and pair.file_path2 == pair.file_path
1812
+ )
1813
+
1814
+ target_insert_fn: Optional[str] = None
1815
+ if same_file_for_hygiene:
1816
+ target_insert_fn = self._deepest_common_ancestry(
1817
+ pair.function1_ancestry, pair.function2_ancestry
1818
+ )
1819
+ if target_insert_fn:
1820
+ # Locate the target function node and its scope analyzer for this file
1821
+ target_func_node = None
1822
+ target_analyzer: Optional[ScopeAnalyzer] = None
1823
+ for fpath, fn, _src, analyzer, _rscope, _cls, _encl, _anc in all_functions:
1824
+ if fpath == pair.file_path and fn.name == target_insert_fn:
1825
+ target_func_node = fn
1826
+ target_analyzer = analyzer
1827
+ break
1828
+ if target_func_node is not None and target_analyzer is not None:
1829
+ func_scope = target_analyzer.node_scopes.get(target_func_node)
1830
+ if func_scope is not None:
1831
+ enclosing_names.update(func_scope.bindings.keys())
1832
+ except Exception:
1833
+ # Best-effort only; if anything goes wrong, proceed with module-level names
1834
+ pass
1835
+
1836
+ # Compute free variables for both blocks (variables used but not defined in each block)
1837
+ # Use block1's free variables to derive parameters for the extracted function,
1838
+ # but validate incomplete lifetimes independently for each block.
1839
+ free_vars1 = (
1840
+ scope_analyzer.get_free_variables(pair.block1_nodes) if scope_analyzer else set()
1841
+ )
1842
+ free_vars2 = (
1843
+ scope_analyzer2.get_free_variables(pair.block2_nodes) if scope_analyzer2 else set()
1844
+ )
1845
+
1846
+ # CRITICAL VALIDATION: Reject proposals with incomplete variable lifetimes
1847
+ # A free variable bound AFTER the block is problematic - we'd be using it before it's defined.
1848
+ # However, free variables bound BEFORE the block are OK - they become parameters.
1849
+ if free_vars1 & bound_after_block1:
1850
+ incomplete_vars = free_vars1 & bound_after_block1
1851
+ if os.getenv("DEBUG_VALIDATION"):
1852
+ print(
1853
+ f" REJECTED: Block1 uses variables defined AFTER the block: {incomplete_vars}"
1854
+ )
1855
+ print(" These variables would be used before they're defined")
1856
+ self._debug_reject("incomplete_lifetime_block1", pair, str(incomplete_vars))
1857
+ return None
1858
+
1859
+ if free_vars2 & bound_after_block2:
1860
+ incomplete_vars = free_vars2 & bound_after_block2
1861
+ if os.getenv("DEBUG_VALIDATION"):
1862
+ print(
1863
+ f" REJECTED: Block2 uses variables defined AFTER the block: {incomplete_vars}"
1864
+ )
1865
+ self._debug_reject("incomplete_lifetime_block2", pair, str(incomplete_vars))
1866
+ return None
1867
+
1868
+ # Find all variables used in augmented assignments in block1
1869
+ # These variables MUST be passed as parameters even if they appear in substitution
1870
+ # because augmented assignments (total += x) READ the variable before writing it
1871
+ aug_finder = AugAssignFinder()
1872
+ for node in pair.block1_nodes:
1873
+ aug_finder.visit(node)
1874
+ aug_assign_vars = aug_finder.aug_assign_targets
1875
+
1876
+ # Handle augmented assignment variables specially
1877
+ # 1) Remove them from substitution so they remain as free variables/parameters
1878
+ # 2) Record the name mapping per block for call generation later
1879
+ aug_assign_param_mappings: Dict[str, Dict[int, str]] = {}
1880
+ params_to_remove = []
1881
+ for param_name, exprs in list(substitution.param_expressions.items()):
1882
+ for block_idx, expr in exprs:
1883
+ if block_idx == 0 and isinstance(expr, ast.Name) and expr.id in aug_assign_vars:
1884
+ if param_name not in aug_assign_param_mappings:
1885
+ aug_assign_param_mappings[param_name] = {}
1886
+ aug_assign_param_mappings[param_name][block_idx] = expr.id
1887
+ if 0 in aug_assign_param_mappings.get(param_name, {}):
1888
+ params_to_remove.append(param_name)
1889
+
1890
+ # Store the mappings in the substitution object for later use (dynamic attribute)
1891
+ if not hasattr(substitution, "aug_assign_mappings"):
1892
+ setattr(substitution, "aug_assign_mappings", {})
1893
+ aug_mappings = cast(Dict[str, Dict[int, str]], getattr(substitution, "aug_assign_mappings"))
1894
+ for param_name, block_mappings in aug_assign_param_mappings.items():
1895
+ if 0 in block_mappings:
1896
+ block1_var_name = block_mappings[0]
1897
+ aug_mappings[block1_var_name] = block_mappings
1898
+
1899
+ # Remove the augmented assignment parameters from substitution
1900
+ for param_name in params_to_remove:
1901
+ del substitution.param_expressions[param_name]
1902
+
1903
+ # Never parameterize entire f-strings: if any parameter maps to a JoinedStr in
1904
+ # the template block (block 0), remove it so the f-string structure is preserved
1905
+ fstring_params = []
1906
+ for param_name, exprs in substitution.param_expressions.items():
1907
+ for block_idx, expr in exprs:
1908
+ if block_idx == 0 and isinstance(expr, ast.JoinedStr):
1909
+ fstring_params.append(param_name)
1910
+ break
1911
+ for param_name in fstring_params:
1912
+ del substitution.param_expressions[param_name]
1913
+
1914
+ # Remove variables that have been parameterized from free_vars
1915
+ # If a variable was parameterized (e.g., 'user' -> '__param_5'),
1916
+ # it's no longer free - it's been replaced by a parameter
1917
+ # EXCEPT: variables in augmented assignments MUST remain free variables
1918
+ parameterized_vars = set()
1919
+ for param_name, exprs in substitution.param_expressions.items():
1920
+ for block_idx, expr in exprs:
1921
+ # Only check the first block (template block)
1922
+ if block_idx == 0 and isinstance(expr, ast.Name):
1923
+ # Don't add to parameterized_vars if it's an augmented assignment target
1924
+ if expr.id not in aug_assign_vars:
1925
+ parameterized_vars.add(expr.id)
1926
+
1927
+ # Initialize working free_vars from block1's perspective (template block)
1928
+ free_vars = set(free_vars1) - parameterized_vars
1929
+
1930
+ # CRITICAL: Check if any free variables are declared global or nonlocal
1931
+ # If a free variable is global/nonlocal, we cannot parameterize it
1932
+ # because you cannot have a parameter that is also declared global/nonlocal
1933
+ # This would create: SyntaxError: name 'x' is parameter and global
1934
+ assert scope_analyzer is not None
1935
+ func1_scope_id = None
1936
+ for node, scope in scope_analyzer.node_scopes.items():
1937
+ if isinstance(node, ast.FunctionDef) and node.name == pair.function1_name:
1938
+ func1_scope_id = scope.scope_id
1939
+ break
1940
+
1941
+ globals_to_declare_in_extracted: Set[str] = set()
1942
+ nonlocals_to_declare_in_extracted: Set[str] = set()
1943
+
1944
+ if func1_scope_id is not None:
1945
+ # Check if any variables relevant to this block are global or nonlocal in this scope
1946
+ global_vars = scope_analyzer.global_vars.get(func1_scope_id, set())
1947
+ nonlocal_vars = scope_analyzer.nonlocal_vars.get(func1_scope_id, set())
1948
+
1949
+ # For parameterization safety: if a free variable is declared global/nonlocal, do not parameterize it
1950
+ problematic = free_vars & (global_vars | nonlocal_vars)
1951
+
1952
+ # Collect assignment targets and explicit decls inside the template blocks
1953
+ v = AssignTargetVisitor()
1954
+ for n in pair.block1_nodes:
1955
+ v.visit(n)
1956
+ for n in pair.block2_nodes:
1957
+ v.visit(n)
1958
+ assigned_names = v.assigned_names
1959
+ declared_global_in_block = v.declared_global_in_block
1960
+ declared_nonlocal_in_block = v.declared_nonlocal_in_block
1961
+
1962
+ # Names that are assigned within the block and are global/nonlocal in the enclosing function
1963
+ # MUST be declared in the extracted helper to preserve assignment semantics,
1964
+ # even if they are not free variables of the original block.
1965
+ assigned_problematic_any = assigned_names & (global_vars | nonlocal_vars)
1966
+
1967
+ # For assigned globals/nonlocals that weren't explicitly declared within the block,
1968
+ # promote the declaration into the extracted function body.
1969
+ globals_to_declare_in_extracted = (
1970
+ assigned_problematic_any & global_vars
1971
+ ) - declared_global_in_block
1972
+ nonlocals_to_declare_in_extracted = (
1973
+ assigned_problematic_any & nonlocal_vars
1974
+ ) - declared_nonlocal_in_block
1975
+
1976
+ # Do not parameterize free variables that are global/nonlocal; let them remain free
1977
+ # so the extracted function references the outer binding.
1978
+ free_vars -= problematic
1979
+
1980
+ # Extract function
1981
+ try:
1982
+ func_def, param_order = self.extractor.extract_function(
1983
+ template_block=pair.block1_nodes,
1984
+ substitution=substitution,
1985
+ free_variables=free_vars,
1986
+ enclosing_names=enclosing_names,
1987
+ is_value_producing=value_prod1,
1988
+ return_variables=list(return_variables_block1),
1989
+ global_decls=(
1990
+ globals_to_declare_in_extracted if globals_to_declare_in_extracted else None
1991
+ ),
1992
+ nonlocal_decls=(
1993
+ nonlocals_to_declare_in_extracted if nonlocals_to_declare_in_extracted else None
1994
+ ),
1995
+ # Use hygienic double-underscore name; engine will prefix underscore for methods.
1996
+ function_name="__extracted_func",
1997
+ )
1998
+ except Exception:
1999
+ return None
2000
+
2001
+ # Check for orphaned variables before proceeding
2002
+ # Need to find the function nodes to check for orphans
2003
+ func1_for_orphans: Optional[FunctionNode] = None
2004
+ func2_for_orphans: Optional[FunctionNode] = None
2005
+ for entry in all_functions:
2006
+ file_path, func = entry[0], entry[1]
2007
+ if file_path == pair.file_path and func.name == pair.function1_name:
2008
+ func1_for_orphans = func
2009
+ if (
2010
+ file_path == (pair.file_path2 or pair.file_path)
2011
+ and func.name == pair.function2_name
2012
+ ):
2013
+ func2_for_orphans = func
2014
+
2015
+ if func1_for_orphans and func2_for_orphans:
2016
+ # Get block indices within their respective function bodies
2017
+ indices1 = self._get_block_indices(func1_for_orphans, pair.block1_nodes)
2018
+ indices2 = self._get_block_indices(func2_for_orphans, pair.block2_nodes)
2019
+
2020
+ if indices1 and indices2:
2021
+ # Get function bodies (skip docstring)
2022
+ body1 = func1_for_orphans.body
2023
+ start_idx1 = 0
2024
+ if (
2025
+ body1
2026
+ and isinstance(body1[0], ast.Expr)
2027
+ and isinstance(body1[0].value, ast.Constant)
2028
+ and isinstance(body1[0].value.value, str)
2029
+ ):
2030
+ start_idx1 = 1
2031
+ body1 = body1[start_idx1:]
2032
+
2033
+ body2 = func2_for_orphans.body
2034
+ start_idx2 = 0
2035
+ if (
2036
+ body2
2037
+ and isinstance(body2[0], ast.Expr)
2038
+ and isinstance(body2[0].value, ast.Constant)
2039
+ and isinstance(body2[0].value.value, str)
2040
+ ):
2041
+ start_idx2 = 1
2042
+ body2 = body2[start_idx2:]
2043
+
2044
+ # Check for orphaned variables in both blocks
2045
+ has_orphans1, orphans1 = has_orphaned_variables(
2046
+ cast(List[ast.AST], body1), indices1
2047
+ )
2048
+ has_orphans2, orphans2 = has_orphaned_variables(
2049
+ cast(List[ast.AST], body2), indices2
2050
+ )
2051
+
2052
+ if has_orphans1 or has_orphans2:
2053
+ # Cannot extract - would create orphaned variable references
2054
+ self._debug_reject("orphaned_variables", pair)
2055
+ return None
2056
+
2057
+ # Generate replacement calls
2058
+ replacements: List[Replacement] = []
2059
+
2060
+ # Map block indices to their return variables
2061
+ return_vars_by_block = {0: list(return_variables_block1), 1: list(return_variables_block2)}
2062
+
2063
+ for block_idx, (block_range, file_path) in enumerate(
2064
+ [
2065
+ (pair.block1_range, pair.file_path),
2066
+ (pair.block2_range, pair.file_path2 or pair.file_path),
2067
+ ]
2068
+ ):
2069
+ try:
2070
+ call_node = self.extractor.generate_call(
2071
+ function_name=func_def.name,
2072
+ block_idx=block_idx,
2073
+ substitution=substitution,
2074
+ param_order=param_order,
2075
+ free_variables=free_vars,
2076
+ is_value_producing=value_prod1,
2077
+ return_variables=return_vars_by_block[block_idx],
2078
+ hygienic_renames=hygienic_renames,
2079
+ )
2080
+ # Guard-rail: validate that the generated call does not reference
2081
+ # undefined names at the call site. This rejects brittle proposals
2082
+ # that leak placeholders (e.g., "__param_1") or invented locals
2083
+ # like "filtered"/"mapped" that are not bound before the block.
2084
+ # Allowed names = variables bound before the block ∪ free variables
2085
+ # (builtins are implicitly allowed by runtime).
2086
+ allowed_before: Set[str]
2087
+ if block_idx == 0:
2088
+ allowed_before = set(bound_before_block1) | set(free_vars1)
2089
+ else:
2090
+ allowed_before = set(bound_before_block2) | set(free_vars2)
2091
+
2092
+ used_in_call = self._get_used_names(call_node)
2093
+ # Whitelist a small set of common builtins used in arguments
2094
+ builtin_whitelist = {
2095
+ "len",
2096
+ "sum",
2097
+ "min",
2098
+ "max",
2099
+ "any",
2100
+ "all",
2101
+ "map",
2102
+ "filter",
2103
+ "sorted",
2104
+ "list",
2105
+ "dict",
2106
+ "set",
2107
+ "range",
2108
+ "int",
2109
+ "float",
2110
+ "str",
2111
+ "bool",
2112
+ "enumerate",
2113
+ "zip",
2114
+ }
2115
+ invalid_names = set()
2116
+ for name in used_in_call:
2117
+ if name == func_def.name:
2118
+ continue # referring to the helper itself is handled elsewhere
2119
+ if name.startswith("__param_"):
2120
+ invalid_names.add(name)
2121
+ continue
2122
+ if name in allowed_before or name in builtin_whitelist:
2123
+ continue
2124
+ invalid_names.add(name)
2125
+
2126
+ if invalid_names:
2127
+ # Reject this proposal as it would introduce undefined names
2128
+ self._debug_reject(
2129
+ "undefined_names_in_call",
2130
+ pair,
2131
+ detail=f"block{block_idx+1}: {sorted(invalid_names)}",
2132
+ )
2133
+ return None
2134
+ # Store file_path and class context
2135
+ class_name = pair.class1_name if block_idx == 0 else pair.class2_name
2136
+ method_info = method_info1 if block_idx == 0 else method_info2
2137
+ replacements.append(
2138
+ Replacement(
2139
+ line_range=block_range,
2140
+ node=call_node,
2141
+ file_path=file_path,
2142
+ class_name=class_name,
2143
+ method_kind=method_info.kind,
2144
+ implicit_param=method_info.implicit_param,
2145
+ )
2146
+ )
2147
+ except Exception:
2148
+ return None
2149
+
2150
+ # Multi-occurrence clustering (same-file): look for additional identical blocks
2151
+ # beyond the initial pair and include them in this proposal as extra replacements.
2152
+ # This helps cases like example1_simple where three functions share the same
2153
+ # validator block; by default pairwise selection would only cover two.
2154
+ try:
2155
+ from .block_signature import extract_block_signature, quick_filter as _qf
2156
+
2157
+ # Only attempt simple clustering for module-level, non-returning validators
2158
+ same_file_ctx = (pair.file_path2 is None) or (pair.file_path2 == pair.file_path)
2159
+ if same_file_ctx and not return_variables_block1 and not return_variables_block2:
2160
+ # Build a set of already covered ranges to avoid duplicates
2161
+ covered = {
2162
+ (pair.file_path, pair.block1_range),
2163
+ (pair.file_path2 or pair.file_path, pair.block2_range),
2164
+ }
2165
+ # Template signature from block1
2166
+ tmpl_sig = extract_block_signature(pair.block1_nodes)
2167
+
2168
+ # Gather candidates from same file functions
2169
+ for entry in all_functions:
2170
+ fpath, fn, _src, analyzerX, _rscopeX, clsX, _enclX, _ancX = (
2171
+ entry if len(entry) >= 8 else (*entry, None, None, None)
2172
+ )
2173
+ if fpath != pair.file_path:
2174
+ continue
2175
+ # Skip the original two functions
2176
+ if fn.name in (pair.function1_name, pair.function2_name):
2177
+ # Still scan, but avoid ranges we've already taken
2178
+ pass
2179
+ # Extract blocks and test quick filter against template
2180
+ for cand_range, cand_nodes in self._extract_code_blocks(fn):
2181
+ if (fpath, cand_range) in covered:
2182
+ continue
2183
+ # Minimum size gate
2184
+ start_line, end_line = cand_range
2185
+ if (end_line - start_line + 1) < self.min_lines:
2186
+ continue
2187
+ cand_sig = extract_block_signature(cand_nodes)
2188
+ if not _qf(tmpl_sig, cand_sig):
2189
+ continue
2190
+ # Try to unify template block with candidate
2191
+ try:
2192
+ subst2 = self.unifier.unify_blocks(
2193
+ [pair.block1_nodes, cand_nodes], [{}, {}]
2194
+ )
2195
+ except Exception:
2196
+ continue
2197
+ if not subst2:
2198
+ continue
2199
+ # Orphan check for candidate within its function body
2200
+ indices = self._get_block_indices(fn, cand_nodes)
2201
+ if indices is None:
2202
+ continue
2203
+ # Skip docstring in body
2204
+ body = self._body_without_docstring(fn.body)
2205
+ has_orph, _orph = has_orphaned_variables(cast(List[ast.AST], body), indices)
2206
+ if has_orph:
2207
+ continue
2208
+ # Generate a call node for the candidate
2209
+ try:
2210
+ call_node2 = self.extractor.generate_call(
2211
+ function_name=func_def.name,
2212
+ block_idx=1,
2213
+ substitution=subst2,
2214
+ param_order=param_order,
2215
+ free_variables=free_vars,
2216
+ is_value_producing=value_prod1,
2217
+ return_variables=[],
2218
+ hygienic_renames=[{}, {}],
2219
+ )
2220
+ except Exception:
2221
+ continue
2222
+ # Validate candidate call-site does not reference undefined names
2223
+ try:
2224
+ used2 = self._get_used_names(call_node2)
2225
+ if any(n.startswith("__param_") for n in used2):
2226
+ # Skip brittle candidate that leaked placeholders
2227
+ continue
2228
+ # Compute names bound before candidate block and free vars within it
2229
+ try:
2230
+ from .assignment_analyzer import (
2231
+ _collect_bindings_and_reassignments as _cbar,
2232
+ )
2233
+ except Exception:
2234
+ _cbar = None # type: ignore
2235
+
2236
+ bound_before_cand: Set[str] = set()
2237
+ if _cbar is not None:
2238
+ reassignX = self._get_assignment_reuse(fn)
2239
+ start_line_cand = cand_range[0]
2240
+ for stmt in fn.body:
2241
+ if hasattr(stmt, "lineno") and stmt.lineno < start_line_cand:
2242
+ bset: Set[str] = set()
2243
+ rset: Set[str] = set()
2244
+ _cbar(stmt, reassignX, bset, rset)
2245
+ bound_before_cand.update(bset)
2246
+ # Treat function params as bound
2247
+ if isinstance(fn, ast.FunctionDef):
2248
+ param_namesX: Set[str] = set()
2249
+ for arg in fn.args.args:
2250
+ param_namesX.add(arg.arg)
2251
+ for arg in getattr(fn.args, "posonlyargs", []) or []:
2252
+ param_namesX.add(arg.arg)
2253
+ for arg in fn.args.kwonlyargs:
2254
+ param_namesX.add(arg.arg)
2255
+ if fn.args.vararg:
2256
+ param_namesX.add(fn.args.vararg.arg)
2257
+ if fn.args.kwarg:
2258
+ param_namesX.add(fn.args.kwarg.arg)
2259
+ bound_before_cand.update(param_namesX)
2260
+ free_vars_cand: Set[str] = set()
2261
+ if analyzerX is not None:
2262
+ try:
2263
+ free_vars_cand = set(analyzerX.get_free_variables(cand_nodes))
2264
+ except Exception:
2265
+ free_vars_cand = set()
2266
+ allowed_cand = bound_before_cand | free_vars_cand
2267
+ builtin_whitelist = {
2268
+ "len",
2269
+ "sum",
2270
+ "min",
2271
+ "max",
2272
+ "any",
2273
+ "all",
2274
+ "map",
2275
+ "filter",
2276
+ "sorted",
2277
+ "list",
2278
+ "dict",
2279
+ "set",
2280
+ "range",
2281
+ "int",
2282
+ "float",
2283
+ "str",
2284
+ "bool",
2285
+ "enumerate",
2286
+ "zip",
2287
+ }
2288
+ invalid2 = set()
2289
+ for name in used2:
2290
+ if name == func_def.name:
2291
+ continue
2292
+ if name in allowed_cand or name in builtin_whitelist:
2293
+ continue
2294
+ invalid2.add(name)
2295
+ if invalid2:
2296
+ # Skip this candidate replacement
2297
+ continue
2298
+ except Exception:
2299
+ # On any validator error, be conservative and skip candidate
2300
+ continue
2301
+ # Append replacement
2302
+ replacements.append(
2303
+ Replacement(
2304
+ line_range=cand_range,
2305
+ node=call_node2,
2306
+ file_path=fpath,
2307
+ class_name=clsX,
2308
+ method_kind=None,
2309
+ implicit_param=None,
2310
+ )
2311
+ )
2312
+ covered.add((fpath, cand_range))
2313
+ except Exception:
2314
+ # Non-fatal: clustering is best-effort only
2315
+ pass
2316
+ # Determine canonical file for extracted function
2317
+ # Default to the first file, but this may change if we insert into an ancestor class
2318
+ canonical_file = pair.file_path
2319
+
2320
+ # Create proposal
2321
+ is_cross_file = pair.file_path2 is not None and pair.file_path != pair.file_path2
2322
+
2323
+ desc = f"Extract common code from {pair.function1_name}"
2324
+ if is_cross_file:
2325
+ assert pair.file_path2 is not None
2326
+ desc += f" ({Path(pair.file_path).name}) and {pair.function2_name} ({Path(pair.file_path2).name})"
2327
+ else:
2328
+ desc += f" and {pair.function2_name}"
2329
+
2330
+ # Default to module-level insertion; when possible insert into deepest common enclosing function.
2331
+ insert_into_class = None
2332
+ insert_into_function = None
2333
+ method_kind_metadata: Optional[Literal["instance", "classmethod", "staticmethod"]] = None
2334
+ method_param_name: Optional[str] = None
2335
+
2336
+ # Consider pairs from the same file even if file_path2 is None (same-file pairing)
2337
+ same_file = (pair.file_path2 is None) or (pair.file_path2 == pair.file_path)
2338
+
2339
+ # Prefer insertion into the function that actually contains BOTH blocks when they come from the
2340
+ # same function (process_user_data & process_admin_data are top-level siblings: no shared ancestry).
2341
+ # If the two functions are the SAME name (duplicates), insert into that function instead of module.
2342
+ if same_file:
2343
+ # Only consider deepest common enclosing function based on ancestry; do NOT use
2344
+ # simple name equality as methods across different classes may share the same
2345
+ # name but are not in the same function scope. Name-equality caused incorrectly
2346
+ # inserting helpers inside one sibling method.
2347
+ if pair.function1_ancestry is not None and pair.function2_ancestry is not None:
2348
+ dce = self._deepest_common_ancestry(
2349
+ pair.function1_ancestry, pair.function2_ancestry
2350
+ )
2351
+ if dce:
2352
+ insert_into_function = dce
2353
+
2354
+ class_plan: Optional[ClassInsertionPlan] = None
2355
+ if insert_into_function is None:
2356
+ class_plan = self._choose_class_insertion(
2357
+ pair,
2358
+ method_info1,
2359
+ method_info2,
2360
+ class_infos,
2361
+ )
2362
+
2363
+ if class_plan is not None:
2364
+ # Refined insertion policy:
2365
+ # 1. If both blocks are from the SAME class -> insert into that class.
2366
+ # 2. If blocks are from DIFFERENT classes and class_plan points to a COMMON ANCESTOR
2367
+ # that is neither of the concrete classes, insert into ancestor (shared visibility).
2368
+ # 3. If class_plan resolves to one of the concrete classes while the other differs ->
2369
+ # fallback to module-level to preserve accessibility (avoid privileging one sibling).
2370
+ same_class = pair.class1_name is not None and pair.class1_name == pair.class2_name
2371
+ target_is_concrete_sibling = (
2372
+ class_plan.class_name in {pair.class1_name, pair.class2_name} and not same_class
2373
+ )
2374
+ if target_is_concrete_sibling:
2375
+ # Helper would become invisible to the other sibling; abort class insertion.
2376
+ class_plan = None
2377
+ else:
2378
+ insert_into_class = class_plan.class_name
2379
+ canonical_file = class_plan.file_path
2380
+ method_kind_metadata = class_plan.method_kind
2381
+ method_param_name = class_plan.implicit_param
2382
+
2383
+ # SAFETY: Avoid refactoring across closures with nonlocal variables for now.
2384
+ # If the containing functions (func1/func2) declare any nonlocal variables, skip this proposal
2385
+ # to preserve known semantics and baseline expectations (e.g., closure_adversarial.py).
2386
+ try:
2387
+ nonlocal_in_func1 = False
2388
+ nonlocal_in_func2 = False
2389
+ if scope_analyzer1 and func1 is not None:
2390
+ scope_id1 = scope_analyzer1.node_scopes.get(func1)
2391
+ if scope_id1:
2392
+ nlv1 = scope_analyzer1.nonlocal_vars.get(scope_id1.scope_id, set())
2393
+ nonlocal_in_func1 = bool(nlv1)
2394
+ elif func1 is not None:
2395
+ nonlocal_in_func1 = self._function_contains_nonlocal(func1)
2396
+
2397
+ if scope_analyzer2 and func2 is not None:
2398
+ scope_id2 = scope_analyzer2.node_scopes.get(func2)
2399
+ if scope_id2:
2400
+ nlv2 = scope_analyzer2.nonlocal_vars.get(scope_id2.scope_id, set())
2401
+ nonlocal_in_func2 = bool(nlv2)
2402
+ elif func2 is not None:
2403
+ nonlocal_in_func2 = self._function_contains_nonlocal(func2)
2404
+
2405
+ if nonlocal_in_func1 or nonlocal_in_func2:
2406
+ self._debug_reject("nonlocal_safety_skip", pair)
2407
+ return None
2408
+ except Exception:
2409
+ # On any analyzer lookup issue, be conservative and proceed without special handling
2410
+ pass
2411
+
2412
+ proposal = RefactoringProposal(
2413
+ file_path=canonical_file,
2414
+ extracted_function=func_def,
2415
+ replacements=replacements, # Now includes file_path
2416
+ description=desc,
2417
+ parameters_count=len(substitution.param_expressions),
2418
+ return_variables=list(return_variables_block1),
2419
+ insert_into_class=insert_into_class,
2420
+ insert_into_function=insert_into_function,
2421
+ method_kind=method_kind_metadata,
2422
+ method_param_name=method_param_name,
2423
+ )
2424
+
2425
+ return proposal
2426
+
2427
+ def apply_refactoring(self, file_path: str, proposal: RefactoringProposal) -> str:
2428
+ """
2429
+ Apply a refactoring proposal to a file.
2430
+
2431
+ Args:
2432
+ file_path: Path to file
2433
+ proposal: Refactoring proposal
2434
+
2435
+ Returns:
2436
+ Modified source code
2437
+ """
2438
+ # All proposals now use multi-file format
2439
+ modified_files = self.apply_refactoring_multi_file(proposal)
2440
+ # Backward-compat: handle tuple return (modified_files, changed_paths)
2441
+ if isinstance(modified_files, tuple):
2442
+ modified_files = modified_files[0]
2443
+ # Return the content for the requested file
2444
+ return modified_files.get(file_path, "")
2445
+
2446
+ def apply_refactoring_multi_file(self, proposal: RefactoringProposal) -> Dict[str, str]:
2447
+ """
2448
+ Apply a cross-file refactoring proposal.
2449
+
2450
+ Args:
2451
+ proposal: Refactoring proposal
2452
+
2453
+ Returns:
2454
+ Dict mapping file paths to modified source code
2455
+ """
2456
+ # Group replacements by file
2457
+ replacements_by_file: Dict[str, List[Replacement]] = {}
2458
+ for repl in proposal.replacements:
2459
+ file_path = repl.file_path or proposal.file_path
2460
+ replacements_by_file.setdefault(file_path, []).append(repl)
2461
+
2462
+ # Ensure canonical file is always processed so extracted helper is emitted
2463
+ replacements_by_file.setdefault(proposal.file_path, [])
2464
+
2465
+ # Determine helper names ONCE to avoid mismatches across files
2466
+ # Capture the original helper name before any renaming, and compute the final name
2467
+ original_helper_name = proposal.extracted_function.name
2468
+ if original_helper_name == "__extracted_func":
2469
+ proposal.extracted_function.name = self._allocate_helper_name(proposal.file_path)
2470
+ elif proposal.insert_into_class and not original_helper_name.startswith("_"):
2471
+ # Preserve user-provided helper names but keep them non-public inside classes
2472
+ proposal.extracted_function.name = f"_{original_helper_name}"
2473
+ final_func_name = proposal.extracted_function.name
2474
+
2475
+ # Process each file
2476
+ modified_files: Dict[str, str] = {}
2477
+
2478
+ for file_path, replacements in replacements_by_file.items():
2479
+ # Read original source
2480
+ with open(file_path, "r", encoding="utf-8") as f:
2481
+ lines = f.readlines()
2482
+
2483
+ # Sort replacements by line number (reverse order)
2484
+ replacements = sorted(replacements, key=lambda r: r.line_range[0], reverse=True)
2485
+
2486
+ # Apply each replacement (reverse sorted prevents earlier line shifts)
2487
+ for repl in replacements:
2488
+ start_line, end_line = repl.line_range
2489
+ replacement_node = copy.deepcopy(repl.node)
2490
+ class_name = repl.class_name
2491
+
2492
+ # Validate line numbers
2493
+ if start_line < 1 or start_line > len(lines):
2494
+ print(
2495
+ f"Warning: Invalid line range {start_line}-{end_line} for {file_path} (file has {len(lines)} lines)"
2496
+ )
2497
+ print("Skipping this replacement")
2498
+ continue
2499
+ if end_line > len(lines):
2500
+ print(
2501
+ f"Warning: End line {end_line} exceeds file length {len(lines)} for {file_path}"
2502
+ )
2503
+ print("Adjusting to end of file")
2504
+ end_line = len(lines)
2505
+
2506
+ # Rewrite call sites for method extraction
2507
+ if proposal.insert_into_class and class_name:
2508
+ replacement_node = self._rewrite_call_for_method(
2509
+ replacement_node,
2510
+ original_helper_name,
2511
+ final_func_name,
2512
+ repl.method_kind or proposal.method_kind,
2513
+ repl.implicit_param or proposal.method_param_name,
2514
+ class_name,
2515
+ )
2516
+ elif proposal.insert_into_function:
2517
+ # Rewrite to local function call (no attribute), but ensure call uses final_func_name
2518
+ # Simple textual AST rewrite: replace original helper name with final
2519
+ if original_helper_name != final_func_name:
2520
+ replacement_node = self._retarget_helper_calls(
2521
+ replacement_node,
2522
+ original_helper_name,
2523
+ final_func_name,
2524
+ )
2525
+ else:
2526
+ # Module-level insertion:
2527
+ # If the helper was renamed (e.g., from '__extracted_func' to 'extracted_func' or custom),
2528
+ # rewrite call sites accordingly.
2529
+ if original_helper_name != final_func_name:
2530
+ replacement_node = self._retarget_helper_calls(
2531
+ replacement_node,
2532
+ original_helper_name,
2533
+ final_func_name,
2534
+ )
2535
+
2536
+ replacement_code = ast.unparse(replacement_node)
2537
+ code_lines = replacement_code.split("\n")
2538
+ indent = self._get_indent(lines[start_line - 1])
2539
+
2540
+ # Build indented replacement block: indent ALL non-empty lines consistently
2541
+ replacement_lines: List[str] = []
2542
+ for line in code_lines:
2543
+ if line.strip():
2544
+ replacement_lines.append(indent + line + "\n")
2545
+ else:
2546
+ replacement_lines.append("\n")
2547
+
2548
+ # Splice into source
2549
+ lines[start_line - 1 : end_line] = replacement_lines
2550
+
2551
+ # Insert helper into canonical file or import into others
2552
+ if file_path == proposal.file_path:
2553
+ if proposal.insert_into_function:
2554
+ fn_insert_info = self._find_function_insert_position_before_body_statements(
2555
+ "".join(lines), proposal.insert_into_function
2556
+ )
2557
+ fn_code = ast.unparse(proposal.extracted_function)
2558
+ fn_lines = [l + "\n" for l in fn_code.split("\n")]
2559
+ if fn_insert_info is None:
2560
+ insert_line = self._find_insert_position(lines)
2561
+ lines[insert_line:insert_line] = fn_lines + ["\n", "\n"]
2562
+ else:
2563
+ insert_at_zero_based, indent = fn_insert_info
2564
+ inner_indent = indent + " "
2565
+ indented: List[str] = []
2566
+ for line in fn_lines:
2567
+ if line.strip():
2568
+ indented.append(inner_indent + line)
2569
+ else:
2570
+ indented.append(line)
2571
+ prefix: List[str] = []
2572
+ if insert_at_zero_based > 0 and lines[insert_at_zero_based - 1].strip():
2573
+ prefix.append("\n")
2574
+ suffix: List[str] = []
2575
+ if (
2576
+ insert_at_zero_based < len(lines)
2577
+ and lines[insert_at_zero_based].strip()
2578
+ ):
2579
+ suffix.append("\n")
2580
+ lines[insert_at_zero_based:insert_at_zero_based] = (
2581
+ prefix + indented + suffix
2582
+ )
2583
+ elif proposal.insert_into_class:
2584
+ # Prepare method signature & decorator
2585
+ method_kind = proposal.method_kind or "instance"
2586
+ self._prepare_extracted_method_signature(
2587
+ proposal.extracted_function,
2588
+ method_kind,
2589
+ proposal.method_param_name,
2590
+ )
2591
+ func_code = ast.unparse(proposal.extracted_function)
2592
+ method_lines = [line + "\n" for line in func_code.split("\n")]
2593
+ insert_info = self._find_class_insert_position(
2594
+ "".join(lines), proposal.insert_into_class
2595
+ )
2596
+ if insert_info is None:
2597
+ insert_line = self._find_insert_position(lines)
2598
+ lines[insert_line:insert_line] = method_lines + ["\n", "\n"]
2599
+ else:
2600
+ insert_line_zero_based, indent = insert_info
2601
+ method_indent = indent + " "
2602
+ indented_method: List[str] = []
2603
+ for line in method_lines:
2604
+ if line.strip():
2605
+ indented_method.append(method_indent + line)
2606
+ else:
2607
+ indented_method.append(line)
2608
+ insert_at = insert_line_zero_based + 1
2609
+ method_prefix: List[str] = []
2610
+ if insert_at > 0 and lines[insert_at - 1].strip():
2611
+ method_prefix.append("\n")
2612
+ method_suffix: List[str] = []
2613
+ if insert_at < len(lines) and lines[insert_at].strip():
2614
+ method_suffix.append("\n")
2615
+ lines[insert_at:insert_at] = method_prefix + indented_method + method_suffix
2616
+ else:
2617
+ func_code = ast.unparse(proposal.extracted_function)
2618
+ func_lines = [line + "\n" for line in func_code.split("\n")]
2619
+ insert_line = self._find_insert_position(lines)
2620
+ lines_to_insert: List[str] = []
2621
+ if insert_line > 0:
2622
+ blank_lines_before = 0
2623
+ check_line = insert_line - 1
2624
+ while check_line >= 0 and not lines[check_line].strip():
2625
+ blank_lines_before += 1
2626
+ check_line -= 1
2627
+ if blank_lines_before < 2:
2628
+ lines_to_insert.extend(["\n"] * (2 - blank_lines_before))
2629
+ lines_to_insert.extend(func_lines)
2630
+ lines_to_insert.extend(["\n", "\n"])
2631
+ lines[insert_line:insert_line] = lines_to_insert
2632
+ else:
2633
+ # Non-canonical file: insert import (module-level only)
2634
+ if not proposal.insert_into_class:
2635
+ from_path = Path(proposal.file_path)
2636
+ to_path = Path(file_path)
2637
+ from pathlib import Path as _P
2638
+
2639
+ common_dir = _P(
2640
+ __import__("os").path.commonpath([str(from_path), str(to_path)])
2641
+ )
2642
+ layout = ProjectLayout.discover(
2643
+ common_dir,
2644
+ prefer_absolute_imports=self.prefer_absolute_imports,
2645
+ pep420_namespace_packages=self.pep420_namespace_packages,
2646
+ )
2647
+ abs_mod = layout.module_name_for(from_path)
2648
+ if abs_mod and layout.prefer_absolute_imports:
2649
+ module_name = abs_mod
2650
+ else:
2651
+ if from_path.parent == to_path.parent:
2652
+ module_name = from_path.stem
2653
+ else:
2654
+ module_name = abs_mod or from_path.stem
2655
+ func_name = proposal.extracted_function.name
2656
+ import_line = f"from {module_name} import {func_name}\n"
2657
+ if not any(import_line.strip() == ln.strip() for ln in lines):
2658
+ import_pos = self._find_import_position(lines)
2659
+ lines.insert(import_pos, import_line)
2660
+
2661
+ modified_files[file_path] = "".join(lines)
2662
+
2663
+ return modified_files
2664
+
2665
+ def _find_import_position(self, lines: List[str]) -> int:
2666
+ """Find position to insert a new import statement."""
2667
+ last_import_line, after_docstring = self._scan_module_docstring_and_imports(lines)
2668
+ return last_import_line if last_import_line > 0 else after_docstring
2669
+
2670
+ def _get_indent(self, line: str) -> str:
2671
+ """Get the indentation of a line."""
2672
+ return line[: len(line) - len(line.lstrip())]
2673
+
2674
+ def _find_insert_position(self, lines: List[str]) -> int:
2675
+ """
2676
+ Find position to insert extracted function at END of file.
2677
+
2678
+ This prevents line number shifts when multiple refactorings are applied,
2679
+ making sequential refactorings more robust.
2680
+ """
2681
+ # Insert at the end of the file
2682
+ # Find the last non-blank line
2683
+ for i in range(len(lines) - 1, -1, -1):
2684
+ if lines[i].strip():
2685
+ # Insert after this line
2686
+ return i + 1
2687
+
2688
+ # Empty file - insert at beginning
2689
+ return 0
2690
+
2691
+ def _find_class_insert_position(
2692
+ self, source: str, class_name: str
2693
+ ) -> Optional[Tuple[int, str]]:
2694
+ """
2695
+ Find insertion position (0-based line index) at end of class body and class indentation.
2696
+
2697
+ Returns (insert_line_index, class_indent_str) or None.
2698
+ """
2699
+ try:
2700
+ tree = ast.parse(source)
2701
+ except SyntaxError:
2702
+ return None
2703
+
2704
+ locator = ClassLocator(source, class_name)
2705
+ locator.visit(tree)
2706
+ return locator.result
2707
+
2708
+ def _find_function_insert_position_before_body_statements(
2709
+ self, source: str, function_name: str
2710
+ ) -> Optional[Tuple[int, str]]:
2711
+ """
2712
+ Find an insertion position (0-based line index) inside the given function BEFORE
2713
+ executable body statements (i.e., after any docstring and after any leading
2714
+ nested defs), along with the function's indentation.
2715
+
2716
+ This ensures the inserted helper is bound before returns/calls are executed.
2717
+ Returns (insert_line_index, function_indent_str) or None if function not found.
2718
+ """
2719
+ try:
2720
+ tree = ast.parse(source)
2721
+ except SyntaxError:
2722
+ return None
2723
+
2724
+ locator = FuncLocator(source, function_name)
2725
+ locator.visit(tree)
2726
+ return locator.result
2727
+
2728
+ def _get_block_indices(
2729
+ self, function: FunctionNode, block_nodes: List[ast.AST]
2730
+ ) -> Optional[Tuple[int, int]]:
2731
+ """
2732
+ Find the indices of a block within a function body.
2733
+
2734
+ Args:
2735
+ function: Function definition
2736
+ block_nodes: Block to find
2737
+
2738
+ Returns:
2739
+ (start_index, end_index) or None if not found
2740
+ """
2741
+ if not block_nodes:
2742
+ return None
2743
+
2744
+ # Get function body (skip docstring)
2745
+ body = self._body_without_docstring(function.body)
2746
+
2747
+ # Match by line numbers
2748
+ first_node = cast(Union[ast.stmt, ast.expr], block_nodes[0])
2749
+ last_node = cast(Union[ast.stmt, ast.expr], block_nodes[-1])
2750
+ block_start_line = first_node.lineno
2751
+ block_end_line = (
2752
+ last_node.end_lineno
2753
+ if hasattr(last_node, "end_lineno") and last_node.end_lineno is not None
2754
+ else last_node.lineno
2755
+ )
2756
+
2757
+ # Find matching range in body
2758
+ for i, stmt in enumerate(body):
2759
+ stmt_start = stmt.lineno
2760
+ stmt_end = stmt.end_lineno if hasattr(stmt, "end_lineno") else stmt.lineno
2761
+
2762
+ if stmt_start == block_start_line:
2763
+ # Found start, now find end
2764
+ for j in range(i, len(body)):
2765
+ stmt_end = (
2766
+ body[j].end_lineno if hasattr(body[j], "end_lineno") else body[j].lineno
2767
+ )
2768
+ if stmt_end == block_end_line:
2769
+ return (i, j)
2770
+
2771
+ return None
2772
+
2773
+ def _are_structurally_similar(
2774
+ self,
2775
+ block1: List[ast.AST],
2776
+ block2: List[ast.AST],
2777
+ threshold: float = DEFAULT_SIMILARITY_THRESHOLD,
2778
+ ) -> bool:
2779
+ """
2780
+ Check if two blocks are structurally similar enough to attempt unification.
2781
+
2782
+ This does a rough structural comparison to filter out obviously different blocks.
2783
+
2784
+ Args:
2785
+ block1: First block
2786
+ block2: Second block
2787
+ threshold: Similarity threshold (0.0 to 1.0, default: 0.6)
2788
+
2789
+ Returns:
2790
+ True if blocks are similar enough
2791
+ """
2792
+ if len(block1) != len(block2):
2793
+ return False
2794
+
2795
+ total_nodes = 0
2796
+ matching_nodes = 0
2797
+
2798
+ for stmt1, stmt2 in zip(block1, block2):
2799
+ # Compare AST structure
2800
+ nodes1 = list(ast.walk(stmt1))
2801
+ nodes2 = list(ast.walk(stmt2))
2802
+
2803
+ # Must have similar number of nodes
2804
+ if abs(len(nodes1) - len(nodes2)) / max(len(nodes1), len(nodes2)) > 0.3:
2805
+ return False
2806
+
2807
+ # Count matching node types
2808
+ types1 = [type(n).__name__ for n in nodes1]
2809
+ types2 = [type(n).__name__ for n in nodes2]
2810
+
2811
+ # Count common types
2812
+ from collections import Counter
2813
+
2814
+ counter1 = Counter(types1)
2815
+ counter2 = Counter(types2)
2816
+
2817
+ common = sum((counter1 & counter2).values())
2818
+ total = max(len(types1), len(types2))
2819
+
2820
+ total_nodes += total
2821
+ matching_nodes += common
2822
+
2823
+ if total_nodes == 0:
2824
+ return False
2825
+
2826
+ similarity = matching_nodes / total_nodes
2827
+ return similarity >= threshold
2828
+
2829
+ def refactor_to_fixed_point(
2830
+ self, file_path: str, max_iterations: int = 10
2831
+ ) -> Tuple[str, int, List[str]]:
2832
+ """
2833
+ Apply refactorings iteratively until a fixed point is reached.
2834
+
2835
+ This method applies refactorings one at a time, re-analyzing after each
2836
+ application. This prevents the sequential corruption bug where applying
2837
+ multiple refactorings at once causes line number misalignment.
2838
+
2839
+ max_iterations semantics:
2840
+ - If max_iterations > 0, stop after at most that many applied refactorings.
2841
+ - If max_iterations <= 0, run until a natural fixed point (no proposals found).
2842
+
2843
+ Args:
2844
+ file_path: Path to file to refactor
2845
+ max_iterations: Maximum iterations to prevent infinite loops
2846
+
2847
+ Returns:
2848
+ Tuple of (final_code, num_refactorings_applied, descriptions)
2849
+ """
2850
+ current_code = Path(file_path).read_text()
2851
+ num_applied = 0
2852
+ descriptions = []
2853
+
2854
+ iteration = 0
2855
+ while True:
2856
+ # Write current code to file for analysis
2857
+ with open(file_path, "w") as f:
2858
+ f.write(current_code)
2859
+
2860
+ # Analyze for refactoring opportunities
2861
+ # Important: invalidate cached analysis for this path so we see latest edits
2862
+ proposals = self.analyze_files([file_path], invalidate_paths=[file_path])
2863
+
2864
+ if not proposals:
2865
+ # Fixed point reached - no more refactorings found
2866
+ break
2867
+
2868
+ # Apply only the first proposal
2869
+ proposal = proposals[0]
2870
+ new_code = self.apply_refactoring(file_path, proposal)
2871
+ # Idempotence guard: if no change, stop to avoid churn
2872
+ if new_code == current_code:
2873
+ break
2874
+ current_code = new_code
2875
+ num_applied += 1
2876
+ descriptions.append(proposal.description)
2877
+
2878
+ iteration += 1
2879
+ if max_iterations > 0 and iteration >= max_iterations:
2880
+ break
2881
+
2882
+ return current_code, num_applied, descriptions
2883
+
2884
+ def refactor_directory_to_fixed_point(
2885
+ self,
2886
+ input_dir: str,
2887
+ output_dir: str,
2888
+ max_iterations: int = 10,
2889
+ progress: str = "tqdm",
2890
+ ) -> Tuple[Dict[str, Tuple[int, List[str]]], str]:
2891
+ """
2892
+ Apply refactorings across a directory (recursively) until a fixed point.
2893
+
2894
+ Unlike the per-file variant, this performs whole-project analysis on every
2895
+ iteration so it can apply BOTH same-file and cross-file proposals. One proposal
2896
+ is applied per iteration, then the directory is re-analyzed, up to max_iterations.
2897
+
2898
+ Args:
2899
+ input_dir: Input directory path
2900
+ output_dir: Output directory path (results are written here)
2901
+ max_iterations: Maximum iterations to prevent infinite loops. If <= 0,
2902
+ run until a natural fixed point (no proposals remain).
2903
+ progress: Progress display mode: 'tqdm' for percentage bar if available,
2904
+ 'auto' fallback to simple inline bar, 'none' disables progress output.
2905
+
2906
+ Returns:
2907
+ (results_dict, termination_reason)
2908
+ termination_reason ∈ {"fixed_point", "iteration_cap"}
2909
+ """
2910
+ from pathlib import Path
2911
+ import shutil
2912
+ import textwrap
2913
+
2914
+ progress_mode, tqdm_wrapper, use_tqdm = self._resolve_progress_backend(progress)
2915
+
2916
+ input_path = Path(input_dir)
2917
+ output_path = Path(output_dir)
2918
+
2919
+ # Create output directory
2920
+ output_path.mkdir(parents=True, exist_ok=True)
2921
+
2922
+ # Copy all files to output directory first (skip if same path)
2923
+ if input_path != output_path:
2924
+ for item in input_path.rglob("*"):
2925
+ if item.is_file():
2926
+ rel_path = item.relative_to(input_path)
2927
+ output_file = output_path / rel_path
2928
+ output_file.parent.mkdir(parents=True, exist_ok=True)
2929
+ shutil.copy2(item, output_file)
2930
+
2931
+ # Aggregate results per file
2932
+ results: Dict[str, Tuple[int, List[str]]] = {}
2933
+
2934
+ def _bump_result(path: str, desc: str) -> None:
2935
+ count, descs = results.get(path, (0, []))
2936
+ results[path] = (count + 1, descs + [desc])
2937
+
2938
+ # Proposal processing state
2939
+ proposal_queue: List[RefactoringProposal] = []
2940
+ iterations = 0
2941
+ total_applied = 0
2942
+ termination_reason = "fixed_point"
2943
+
2944
+ # Timing / ETA state (for heuristic ETA when total unknown)
2945
+ import time as _time
2946
+
2947
+ per_proposal_durations: List[float] = []
2948
+
2949
+ # Progress helpers -------------------------------------------------
2950
+ progress_bar = None
2951
+
2952
+ # Fallback inline bar (only when not using tqdm and not in detail/none)
2953
+ def _fallback_bar(applied: int, queued: int, phase: str, desc: str) -> None:
2954
+ # Suppress inline fallback bar when tqdm is selected or active, or in 'none'/'detail' modes
2955
+ if progress_mode in ("none", "detail", "tqdm") or use_tqdm:
2956
+ return
2957
+ try:
2958
+ denom = max(applied + queued, 1)
2959
+ pct = int((applied / denom) * 100)
2960
+ bar = self._render_inline_bar(pct, bar_len=32)
2961
+ short = desc if len(desc) <= 48 else desc[:45] + "..."
2962
+ print(
2963
+ f"\r[towel] {phase:<10} [{bar}] {pct:3d}% | applied={applied} queued={queued} | {short}",
2964
+ end="",
2965
+ flush=True,
2966
+ )
2967
+ except Exception:
2968
+ pass
2969
+
2970
+ def _update_progress_postfix(applied: int, queued: int) -> None:
2971
+ """Keep tqdm postfix updates consistent (see docs/DRY_RUN_2025-11-28.md)."""
2972
+ if not (use_tqdm and progress_bar is not None):
2973
+ return
2974
+ try:
2975
+ progress_bar.set_postfix({"A": applied, "Q": queued}, refresh=True)
2976
+ except Exception:
2977
+ pass
2978
+
2979
+ def _apply_proposal_and_refresh_queue(
2980
+ proposal: RefactoringProposal, queue: List[RefactoringProposal]
2981
+ ) -> List[RefactoringProposal]:
2982
+ """Apply proposal, update caches, and refresh queue (DRY helper)."""
2983
+ result = self.apply_refactoring_multi_file(proposal)
2984
+ if isinstance(result, tuple):
2985
+ modified_files, changed_paths = result
2986
+ else:
2987
+ modified_files = result
2988
+ changed_paths = list(modified_files.keys())
2989
+
2990
+ for fpath, content in modified_files.items():
2991
+ with open(fpath, "w", encoding="utf-8") as fh:
2992
+ fh.write(content)
2993
+ _bump_result(fpath, proposal.description)
2994
+
2995
+ if hasattr(self, "invalidate_paths") and changed_paths:
2996
+ try:
2997
+ self.invalidate_paths(changed_paths)
2998
+ except Exception:
2999
+ pass
3000
+
3001
+ if changed_paths:
3002
+ changed_set = set(map(str, changed_paths))
3003
+ queue = [
3004
+ p
3005
+ for p in queue
3006
+ if not any(
3007
+ (rep.file_path or p.file_path) in changed_set for rep in p.replacements
3008
+ )
3009
+ ]
3010
+
3011
+ try:
3012
+ localized = self.analyze_files(
3013
+ list(changed_paths), invalidate_paths=list(changed_paths)
3014
+ )
3015
+ if localized:
3016
+ localized = filter_overlapping_proposals(localized)
3017
+ localized = [
3018
+ p
3019
+ for p in localized
3020
+ if any(
3021
+ (rep.file_path or p.file_path) in changed_set
3022
+ for rep in p.replacements
3023
+ )
3024
+ ]
3025
+ if localized:
3026
+ queue = localized + queue
3027
+ _detail(f"Localized +{len(localized)} follow-up(s)")
3028
+ _fallback_bar(
3029
+ total_applied,
3030
+ len(queue),
3031
+ "localized",
3032
+ f"+{len(localized)} follow-ups",
3033
+ )
3034
+ except Exception:
3035
+ pass
3036
+
3037
+ return queue
3038
+
3039
+ # Note: We defer tqdm progress bar creation until we have proposals to apply.
3040
+ # This avoids an early line like "analyzing: 0it" with unknown totals.
3041
+
3042
+ def _detail(msg: str) -> None:
3043
+ if progress_mode == "detail":
3044
+ print(f"[towel] {msg}")
3045
+
3046
+ # Main loop -------------------------------------------------------
3047
+ while True:
3048
+ if not proposal_queue:
3049
+ # Global analysis pass
3050
+ if progress_mode != "none":
3051
+ try:
3052
+ file_count = sum(1 for _ in output_path.rglob("*.py"))
3053
+ _detail(f"Analyzing {file_count} file(s)...")
3054
+ except Exception:
3055
+ pass
3056
+ # Show pairing progress during global analysis if user requested progress bars.
3057
+ analysis_progress_flag = (
3058
+ progress_mode if progress_mode in ("tqdm", "auto") else "none"
3059
+ )
3060
+ proposals = self.analyze_directory(
3061
+ str(output_path), recursive=True, verbose=False, progress=analysis_progress_flag
3062
+ )
3063
+ if not proposals:
3064
+ # Fixed point reached
3065
+ if use_tqdm and progress_bar is not None:
3066
+ # Ensure a clean newline so the last line doesn't meld with following prints
3067
+ try:
3068
+ progress_bar.refresh()
3069
+ progress_bar.close()
3070
+ except Exception:
3071
+ pass
3072
+ else:
3073
+ if progress_mode not in ("none", "detail") and not use_tqdm:
3074
+ print() # finish inline bar line
3075
+ break
3076
+ proposal_queue = filter_overlapping_proposals(proposals)
3077
+ _detail(f"Discovered {len(proposal_queue)} proposal(s)")
3078
+ if progress_mode == "detail":
3079
+ for i, p in enumerate(proposal_queue[:25], 1): # cap verbose listing
3080
+ short = textwrap.shorten(p.description, width=100, placeholder="...")
3081
+ print(f" {i:2d}. {short}")
3082
+ if len(proposal_queue) > 25:
3083
+ print(f" ... {len(proposal_queue)-25} more")
3084
+ _fallback_bar(total_applied, len(proposal_queue), "discovered", "proposals queued")
3085
+ if use_tqdm and progress_bar is None:
3086
+ # Lazily create tqdm now that we have proposals to apply
3087
+ assert tqdm_wrapper is not None
3088
+ try:
3089
+ total_known = max_iterations > 0
3090
+ # Use leave=False so subsequent prints don't duplicate the bar line.
3091
+ progress_bar = tqdm_wrapper(
3092
+ total=max_iterations if total_known else None,
3093
+ desc="apply",
3094
+ unit="it",
3095
+ dynamic_ncols=True,
3096
+ leave=False,
3097
+ )
3098
+ queued_ct = len(proposal_queue)
3099
+ _update_progress_postfix(0, queued_ct)
3100
+ except Exception:
3101
+ progress_bar = None
3102
+
3103
+ if not proposal_queue:
3104
+ break
3105
+
3106
+ proposal = self._pop_next_proposal(proposal_queue)
3107
+ if proposal is None:
3108
+ break
3109
+ iter_start = _time.time()
3110
+ last_desc = proposal.description
3111
+ # Suppress separate applying log line when tqdm active to avoid duplicate lines
3112
+ if not (use_tqdm and progress_bar is not None):
3113
+ _fallback_bar(
3114
+ total_applied, len(proposal_queue), "apply", f"#{iterations+1}: {last_desc}"
3115
+ )
3116
+
3117
+ # Apply proposal
3118
+ proposal_queue = _apply_proposal_and_refresh_queue(proposal, proposal_queue)
3119
+
3120
+ # Record duration for this iteration (include localized follow-up analysis time)
3121
+ per_proposal_durations.append(_time.time() - iter_start)
3122
+ iterations += 1
3123
+ total_applied += 1
3124
+ if use_tqdm and progress_bar is not None:
3125
+ try:
3126
+ progress_bar.update(1)
3127
+ queued_ct = len(proposal_queue)
3128
+ except Exception:
3129
+ pass
3130
+ else:
3131
+ _update_progress_postfix(total_applied, queued_ct)
3132
+ else:
3133
+ _fallback_bar(
3134
+ total_applied, len(proposal_queue), "applied", f"#{iterations}: {last_desc}"
3135
+ )
3136
+
3137
+ if max_iterations > 0 and iterations >= max_iterations:
3138
+ termination_reason = "iteration_cap"
3139
+ if use_tqdm and progress_bar is not None:
3140
+ progress_bar.close()
3141
+ else:
3142
+ if progress_mode not in ("none", "detail") and not use_tqdm:
3143
+ print()
3144
+ break
3145
+
3146
+ return results, termination_reason
3147
+
3148
+ # Optional analysis cache invalidation hook used by directory fixed-point runner
3149
+ def invalidate_paths(self, paths: List[str]) -> None: # pragma: no cover - simple cache hook
3150
+ try:
3151
+ from . import pipeline as _pipeline # local import to avoid cycles at module load time
3152
+
3153
+ for p in paths:
3154
+ if hasattr(_pipeline, "_analysis_cache"):
3155
+ cache = cast(Dict[str, Any], getattr(_pipeline, "_analysis_cache"))
3156
+ cache.pop(p, None)
3157
+ except Exception:
3158
+ # Non-fatal; best-effort invalidation only
3159
+ pass
3160
+
3161
+
3162
+ def _refactor_engine_helper_2(self, all_functions, block_pairs, class_infos, progress, verbose):
3163
+ return self._evaluate_pairs_serial(
3164
+ block_pairs, all_functions, class_infos, verbose=verbose, progress=progress
3165
+ )
3166
+
3167
+
3168
+ # Utility functions for overlap filtering
3169
+
3170
+
3171
+ def get_affected_lines(proposal: RefactoringProposal) -> Set[Tuple[str, int]]:
3172
+ """
3173
+ Get all (file_path, line_number) tuples affected by a proposal.
3174
+
3175
+ This is used for overlap detection - two proposals overlap if they
3176
+ affect any of the same lines in the same file.
3177
+
3178
+ Args:
3179
+ proposal: A refactoring proposal
3180
+
3181
+ Returns:
3182
+ Set of (file_path, line_number) tuples that would be modified
3183
+ """
3184
+ affected: Set[Tuple[str, int]] = set()
3185
+ for repl in proposal.replacements:
3186
+ file_path = repl.file_path or proposal.file_path
3187
+ start_line, end_line = repl.line_range
3188
+ for line_num in range(start_line, end_line + 1):
3189
+ affected.add((file_path, line_num))
3190
+
3191
+ return affected
3192
+
3193
+
3194
+ def filter_overlapping_proposals(proposals: List[RefactoringProposal]) -> List[RefactoringProposal]:
3195
+ """
3196
+ Filter proposals to remove overlaps, keeping the best ones.
3197
+
3198
+ When multiple proposals affect overlapping lines, this function selects
3199
+ a subset of non-overlapping proposals. Larger proposals (affecting more
3200
+ lines) are preferred over smaller ones.
3201
+
3202
+ Strategy:
3203
+ 1. Sort proposals by size (total lines affected), largest first
3204
+ 2. Greedily select proposals that don't overlap with already-selected ones
3205
+
3206
+ Args:
3207
+ proposals: List of refactoring proposals
3208
+
3209
+ Returns:
3210
+ List of non-overlapping proposals, sorted by size (largest first)
3211
+
3212
+ Example:
3213
+ If proposals affect lines [1-7], [1-6], and [2-7], only [1-7]
3214
+ would be selected as it's the largest and the others overlap with it.
3215
+ """
3216
+ if not proposals:
3217
+ return []
3218
+
3219
+ def proposal_size(p: RefactoringProposal) -> int:
3220
+ """Calculate total lines affected by a proposal."""
3221
+ total_lines = 0
3222
+ for repl in p.replacements:
3223
+ start_line, end_line = repl.line_range
3224
+ total_lines += end_line - start_line + 1
3225
+ return total_lines
3226
+
3227
+ # Optional debug diagnostics: env flag
3228
+ import os as _os
3229
+
3230
+ _debug_overlap = bool(_os.getenv("DEBUG_OVERLAP_FILTER"))
3231
+
3232
+ # Helpers for deterministic ordering and interval extraction
3233
+ def first_span(p: RefactoringProposal) -> Tuple[str, int]:
3234
+ if not p.replacements:
3235
+ return (p.file_path, 0)
3236
+ starts = [r.line_range[0] for r in p.replacements]
3237
+ return (p.file_path, min(starts))
3238
+
3239
+ # Map proposals to affected lines and per-file convex-hull intervals
3240
+ prop_affected: Dict[int, Set[Tuple[str, int]]] = {}
3241
+ prop_files: Dict[int, Set[str]] = {}
3242
+ per_file_interval: Dict[int, Dict[str, Tuple[int, int]]] = {}
3243
+
3244
+ for idx, p in enumerate(proposals):
3245
+ affected = get_affected_lines(p)
3246
+ prop_affected[idx] = affected
3247
+ files: Set[str] = set(fp for (fp, _ln) in affected)
3248
+ prop_files[idx] = files
3249
+ per_file_interval[idx] = {}
3250
+ # Build convex hull interval per file for MWIS approximation
3251
+ by_file: Dict[str, List[int]] = {}
3252
+ for fp, ln in affected:
3253
+ by_file.setdefault(fp, []).append(ln)
3254
+ for fp, lines in by_file.items():
3255
+ per_file_interval[idx][fp] = (min(lines), max(lines))
3256
+
3257
+ # Stage 1: Optimal non-overlapping selection within single-file proposals using MWIS
3258
+ selected_indices: Set[int] = set()
3259
+ used_lines: Set[Tuple[str, int]] = set()
3260
+
3261
+ # Group single-file proposals by that file
3262
+ by_primary_file: Dict[str, List[int]] = {}
3263
+ for idx, files in prop_files.items():
3264
+ if len(files) == 1:
3265
+ fp = next(iter(files))
3266
+ by_primary_file.setdefault(fp, []).append(idx)
3267
+
3268
+ def run_weighted_interval_scheduling(file_path: str, indices: List[int]) -> List[int]:
3269
+ # Build items: (start, end, weight, idx)
3270
+ items: List[Tuple[int, int, int, int, Tuple[str, int]]] = []
3271
+ for idx in indices:
3272
+ start, end = per_file_interval[idx][file_path]
3273
+ weight = proposal_size(proposals[idx])
3274
+ items.append((start, end, weight, idx, first_span(proposals[idx])))
3275
+
3276
+ # Sort by end then tie-breaker to stabilize
3277
+ items.sort(key=lambda t: (t[1], t[0], -t[2], t[4]))
3278
+
3279
+ n = len(items)
3280
+ if n == 0:
3281
+ return []
3282
+
3283
+ # Precompute p[j]: rightmost non-overlapping interval index before j
3284
+ ends = [it[1] for it in items]
3285
+ starts = [it[0] for it in items]
3286
+ p = [-1] * n
3287
+ j = 0
3288
+ for j in range(n):
3289
+ # binary search for last i with ends[i] < starts[j]
3290
+ lo, hi = 0, j - 1
3291
+ last = -1
3292
+ while lo <= hi:
3293
+ mid = (lo + hi) // 2
3294
+ if ends[mid] < starts[j]:
3295
+ last = mid
3296
+ lo = mid + 1
3297
+ else:
3298
+ hi = mid - 1
3299
+ p[j] = last
3300
+
3301
+ # DP arrays
3302
+ dp = [0] * n
3303
+ take = [False] * n
3304
+ for j in range(n):
3305
+ wj = items[j][2]
3306
+ without = dp[j - 1] if j > 0 else 0
3307
+ withj = wj + (dp[p[j]] if p[j] != -1 else 0)
3308
+ if withj > without:
3309
+ dp[j] = withj
3310
+ take[j] = True
3311
+ elif withj == without:
3312
+ # Tie-breaker: prefer earlier ending interval set implicitly
3313
+ dp[j] = without
3314
+ take[j] = False
3315
+ else:
3316
+ dp[j] = without
3317
+ take[j] = False
3318
+
3319
+ # Reconstruct
3320
+ sel: List[int] = []
3321
+ j = n - 1
3322
+ while j >= 0:
3323
+ if take[j]:
3324
+ sel.append(items[j][3])
3325
+ j = p[j]
3326
+ else:
3327
+ j -= 1
3328
+ sel.reverse()
3329
+ if _debug_overlap:
3330
+ try:
3331
+ print(
3332
+ f"OVERLAP_OPTIMAL file={file_path} selected={len(sel)} total_weight={dp[n-1]} candidates={n}"
3333
+ )
3334
+ except Exception:
3335
+ pass
3336
+ return sel
3337
+
3338
+ for fp, idxs in by_primary_file.items():
3339
+ chosen = run_weighted_interval_scheduling(fp, idxs)
3340
+ for idx in chosen:
3341
+ if idx not in selected_indices:
3342
+ selected_indices.add(idx)
3343
+ used_lines.update(prop_affected[idx])
3344
+
3345
+ # Stage 2: Greedy add for remaining proposals (multi-file or leftover), respecting used_lines
3346
+ def sort_key(p: RefactoringProposal) -> Tuple[int, Tuple[str, int]]:
3347
+ return (-(proposal_size(p)), first_span(p))
3348
+
3349
+ remaining = [i for i in range(len(proposals)) if i not in selected_indices]
3350
+ remaining_sorted = sorted(remaining, key=lambda i: sort_key(proposals[i]))
3351
+
3352
+ selected: List[RefactoringProposal] = [proposals[i] for i in selected_indices]
3353
+
3354
+ for idx in remaining_sorted:
3355
+ affected = prop_affected[idx]
3356
+ intersection = affected & used_lines
3357
+ if not intersection:
3358
+ selected.append(proposals[idx])
3359
+ used_lines.update(affected)
3360
+ elif _debug_overlap:
3361
+ by_file2: Dict[str, List[int]] = {}
3362
+ for fp, ln in intersection:
3363
+ by_file2.setdefault(fp, []).append(ln)
3364
+ parts = [
3365
+ f"{fp}:{min(lines)}-{max(lines)} ({len(lines)} lines)"
3366
+ for fp, lines in by_file2.items()
3367
+ ]
3368
+ try:
3369
+ print(
3370
+ "OVERLAP_DROP: size=",
3371
+ proposal_size(proposals[idx]),
3372
+ " first=",
3373
+ first_span(proposals[idx]),
3374
+ " because intersects ",
3375
+ "; ".join(parts),
3376
+ )
3377
+ except Exception:
3378
+ pass
3379
+
3380
+ # Return selected sorted by size descending for external stability
3381
+ return sorted(selected, key=lambda p: (-(proposal_size(p)), first_span(p)))